From 9529c99f5ebd5469942feb2c83a785894f48522b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 16 Nov 2026 08:47:28 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=91=20fix:=20Harden=20Remote=20SDK=20L?= =?UTF-8?q?ifecycle=20and=20Recovery=20(#18213)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: propagate remote SDK tool invocation cancellation * fix: harden remote SDK drain, replay, and state parity --- docs/sdk/agents.md | 21 +- examples/sdk-runtime/README.md | 38 ++- examples/sdk-runtime/server.cjs | 1 + .../api/src/agents/remoteSdk/bridge.spec.ts | 320 +++++++++++++++++- packages/api/src/agents/remoteSdk/client.ts | 61 +++- .../src/agents/remoteSdk/hostTools.spec.ts | 132 +++++++- .../api/src/agents/remoteSdk/hostTools.ts | 45 ++- .../api/src/agents/remoteSdk/peer.spec.ts | 40 +++ packages/api/src/agents/remoteSdk/peer.ts | 50 ++- .../src/agents/remoteSdk/recovery/client.ts | 128 ++++--- .../journal.cache_integration.spec.ts | 19 ++ .../src/agents/remoteSdk/recovery/journal.ts | 1 + .../api/src/agents/remoteSdk/recovery/log.ts | 42 +-- .../src/agents/remoteSdk/recovery/redis.ts | 19 +- .../runtime.cache_integration.spec.ts | 238 ++++++++++++- .../src/agents/remoteSdk/recovery/server.ts | 320 ++++++++++-------- .../agents/remoteSdk/recovery/socket.spec.ts | 121 +++++++ .../src/agents/remoteSdk/recovery/socket.ts | 202 +++++++---- packages/api/src/agents/remoteSdk/server.ts | 18 +- .../api/src/agents/remoteSdk/snapshot.spec.ts | 38 +++ packages/api/src/agents/remoteSdk/snapshot.ts | 76 +++++ packages/api/src/agents/remoteSdk/types.ts | 3 + packages/data-provider/src/config.spec.ts | 7 + packages/data-provider/src/config.ts | 9 + 24 files changed, 1612 insertions(+), 337 deletions(-) create mode 100644 packages/api/src/agents/remoteSdk/recovery/socket.spec.ts create mode 100644 packages/api/src/agents/remoteSdk/snapshot.spec.ts create mode 100644 packages/api/src/agents/remoteSdk/snapshot.ts diff --git a/docs/sdk/agents.md b/docs/sdk/agents.md index 24b931d223..8c6ea9d859 100644 --- a/docs/sdk/agents.md +++ b/docs/sdk/agents.md @@ -134,7 +134,11 @@ Transport limits on each `remote` registration are `connectTimeoutMs` (10000), ` (60000), `maxRunMs` (600000), `maxFrameBytes` (1048576), and `maxBufferedBytes` (4194304). Checkpoint history lists preserve filters/cursors with a protocol page cap of 32 entries. Unresolved secret environment references fail closed rather than becoming signing keys. -Oversized events/checkpoints/results fail closed; raise limits explicitly for larger workloads. +Individual events/checkpoints remain bounded by `maxFrameBytes`. Final content/messages/state +are transferred in ordered chunks; their total serialized snapshot is bounded by `maxBufferedBytes`. +The terminal receipt is accepted only after the complete snapshot arrives. Oversized payloads fail +closed; raise limits explicitly for larger workloads. Live step inspection and reconnect projections +read the same canonical step objects as the streamed event handlers. Worker admission limits are configurable through `createRemoteSdkServer` options. Authorization receives an abort signal; pending authorizations count against admission until they settle, even after disconnect, and shutdown aborts/closes pending handshakes. Honor that signal in async policy @@ -179,7 +183,7 @@ transport success alone is insufficient to mark a product migration complete. ### Host tool execution With SDK 4.2.4+, a host agent's schema tools bind to its existing authenticated tool handler. -The bridge negotiates `hostTools: 1` before starting the worker; an older worker cannot silently +The bridge negotiates `hostTools: 2` before starting the worker; an older worker cannot silently ignore required host tools. Factories install `input.hostTools.toolDefinitions` on the matching SDK agent; supplied config already contains the execution handler. Missing definitions fail before graph execution. Worker-owned direct tools may use `graphTools` alongside host schemas. @@ -200,6 +204,19 @@ control, checkpoint and notification RPCs; a healthy long-running tool must not because it exceeds that shorter control timeout. The run's cancellation/deadline still aborts both sides cooperatively. +Worker tool-call cancellation is scoped to its RPC, including calls still queued behind events. +The host combines that signal with the run signal and waits for the handler's actual async cleanup +before reconciling session state and replying. Late results are suppressed. A Stop request propagates +immediately, but execution ownership is retained until cleanup drains; noncooperative tool code cannot +be forcibly stopped or truthfully reported as drained. RPC deadlines still bound wire waits; session +notifications cannot be delivered after the connection/deadline is lost. Cancellation frames use the +same recovery journal as other traffic and are delivered after reattachment, subject to its limits. + +Upgrade host and worker together: transport capability 2 is required for every remote run, and +host-tool capability 2 is additionally required for host tools. Incompatible releases reject before +invoking the runtime factory. The saved registration protocol remains version 1; the negotiated +transport capability independently versions its frames. + Caller-owned function tools (`clientDelegatedToolNames`) are not host tools. Remote SDK execution rejects that configuration until an explicit ownership/continuation adapter is available; their schemas must never be reinterpreted as permission to execute on the host. diff --git a/examples/sdk-runtime/README.md b/examples/sdk-runtime/README.md index cbf527695f..e7a50dfb24 100644 --- a/examples/sdk-runtime/README.md +++ b/examples/sdk-runtime/README.md @@ -49,10 +49,13 @@ Enable the opt-in recovery configuration below to reattach the same live Run and events without repeating execution. Approval resumes rebuild a fresh worker Run against host-owned checkpoints, without sharing database credentials. -Remote runtimes execute their own operator-installed tools. The bridge deliberately does not -forward host provider credentials, MCP tokens, tool registries, or arbitrary host tool-execution -requests. Configure domain data access inside the trusted worker and authorize it using the -signed caller identity. Host policy hooks still gate SDK tool execution. +Remote runtimes can execute their own operator-installed tools and the saved root agent's +authorized host schema tools. Install `input.hostTools.toolDefinitions` on the SDK agent as the +examples do; the supplied SDK handler dispatches those tools on the host. Provider credentials +and MCP tokens remain host-owned. Configure worker-owned domain access separately and authorize +it using the signed caller identity. Host policy hooks still gate SDK tool execution. +Host-tool capability 2 requires matching host and worker upgrades and propagates per-invocation +cancellation without closing the run's socket; older versions fail before host-tool execution. ## Measuring transport overhead @@ -78,9 +81,9 @@ Enable `remote.recovery: {}` in the host registration, then start the example wi Both sides must run the new bridge version. The worker owns this Redis connection; it never receives the LibreChat database credentials. `SDK_RUNTIME_JOURNAL_PREFIX` separates deployments. -The production API is `createRemoteSdkServer({recovery:{journal,maxDisconnectMs}, ...})`. +The production API is `createRemoteSdkServer({recovery:{journal,maxDisconnectMs,ownerId}, ...})`. `createRedisRuntimeJournal(redis, limits)` accepts an operator-owned Redis-compatible `eval` -client and explicit prefix, retention, lease, frame, byte and batch limits. The example values +client (optionally `evalsha` to avoid resending Lua on every operation) and explicit prefix, retention, lease, frame, byte and batch limits. The example values are illustrative: choose retention and Redis persistence/replication for your data policy. The host YAML defaults are a 15-second disconnect window, 250-ms retry delay and 64-MiB request-local outgoing journal. Keep both sides' disconnect windows aligned and shorter than @@ -92,9 +95,30 @@ A completed/paused terminal result is committed atomically with its terminal jou then published. It can replay after a worker restart. A restart of an **in-flight** graph instead replays committed output and fails safely once the old lease expires: arbitrary side effects cannot be resumed safely by retrying execution. Live reconnect requires routing to the same -worker process. Host process restart adoption and cross-worker live execution migration are +worker process. For a worker pool, set each worker's stable `SDK_RUNTIME_OWNER_ID` and configure +operator-owned direct routes on the host: + +```yaml +recovery: + routes: + worker-a: wss://worker-a.internal/sdk + worker-b: wss://worker-b.internal/sdk +``` + +The initial connection may use the pool URL. After its authenticated ready handshake, the host +reconnects directly to that allowlisted owner, binding the owner ID in the signed reconnect claim. +A worker cannot supply an arbitrary destination; missing routes, missing owner identities with a +route map, and mismatched owners fail closed. Owner identities and routes must remain stable for +the lifetime of a session. A route must identify one worker, not another balanced pool. Single-worker +deployments can omit both the identity and map. Host process restart adoption and cross-worker live execution migration are not supported. The existing host checkpointer still supports explicit approval continuation. +Recovery batches queued frames within the configured journal byte/frame limits, persists cumulative +ACKs separately from control-frame dispatch, and uses EVALSHA with a NOSCRIPT-only fallback when +supported. Writes still commit before publication; cancellation does not wait behind ACK writes. +Final snapshots use bounded ordered chunks and an atomic terminal receipt, including offline +completion/replay. The host's `maxBufferedBytes` bounds snapshot reassembly as well as queued traffic. + Journal keys retain sensitive event/checkpoint data until their TTL; delivery acknowledgements do not delete history. Operators must restrict and encrypt Redis access, configure durability, and integrate `journal.purge({sessionId,scope})` with any earlier deletion requirements. Automatic diff --git a/examples/sdk-runtime/server.cjs b/examples/sdk-runtime/server.cjs index 6794d618fb..dea1f11c4f 100644 --- a/examples/sdk-runtime/server.cjs +++ b/examples/sdk-runtime/server.cjs @@ -24,6 +24,7 @@ const recovery = redis maxBatchBytes: 4194304, }), maxDisconnectMs: 15000, + ownerId: process.env.SDK_RUNTIME_OWNER_ID, } : undefined; const runtime = createRemoteSdkServer({ diff --git a/packages/api/src/agents/remoteSdk/bridge.spec.ts b/packages/api/src/agents/remoteSdk/bridge.spec.ts index b2d4cf9688..191cd33e30 100644 --- a/packages/api/src/agents/remoteSdk/bridge.spec.ts +++ b/packages/api/src/agents/remoteSdk/bridge.spec.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; -import WebSocket from 'ws'; import jwt from 'jsonwebtoken'; import { randomUUID } from 'node:crypto'; +import WebSocket, { WebSocketServer } from 'ws'; import { tool } from '@librechat/agents/langchain/tools'; import { FakeChatModel as TestModel } from '@langchain/core/utils/testing'; import { HumanMessage, AIMessageChunk } from '@librechat/agents/langchain/messages'; @@ -19,6 +19,7 @@ import type { RemoteSdkServerOptions } from './server'; import type { RemoteSdkFactoryInput } from './types'; import { createRemoteSdkServer } from './server'; import { createRemoteSdkRun } from './client'; +import { createSdkPeer } from './peer'; const secret = 'sdk-runtime-test-secret-at-least-32-characters'; const identity = { @@ -696,6 +697,45 @@ describe('remote SDK runtime bridge', () => { await runtime.close(); } }); + it('rejects workers without cancellable host tools before sending a run', async () => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await new Promise((resolve) => server.once('listening', resolve)); + const methods: string[] = []; + server.on('connection', (socket) => + createSdkPeer(socket, { + maxFrameBytes: 65536, + maxBufferedBytes: 131072, + timeoutMs: 2000, + onClose: () => undefined, + onFrame: async () => undefined, + onCall: async (method) => { + methods.push(method); + return { hostTools: 1 }; + }, + }), + ); + try { + const config = hostConfig(); + config.graphConfig = { + type: 'standard', + llmConfig: { provider: Providers.OPENAI }, + toolDefinitions: [{ name: 'inspect' }], + }; + config.customHandlers = { [GraphEvents.ON_TOOL_EXECUTE]: { handle: jest.fn() } }; + const run = await createRemoteSdkRun({ + config, + identity, + remote: { url: `ws://127.0.0.1:${(server.address() as AddressInfo).port}`, secret }, + }); + await expect(run.processStream({ messages: [] }, invocation())).rejects.toThrow( + 'SDK runtime execution failed', + ); + expect(methods).toEqual(['capabilities']); + } finally { + for (const socket of server.clients) socket.terminate(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); it('executes authorized host tools and delivers a fast result before the batch settles', async () => { const definitions = [ { @@ -796,3 +836,281 @@ describe('remote SDK runtime bridge', () => { } }); }); + +it('transfers a terminal snapshot larger than a frame without repeating tool effects', async () => { + const diagnostics: string[] = []; + let workerRun: Awaited> | undefined; + let workerContent: unknown; + let workerMessages: unknown; + let effects = 0; + let largestEvent = 0; + const runtime = await fixture( + async (input) => { + const run = await Run.create({ + ...input.config, + returnContent: true, + tokenCounter: () => 1, + graphConfig: { + type: 'standard', + agents: [ + { + agentId: input.identity.agentId, + provider: Providers.OPENAI, + tools: [], + maxToolResultChars: 50000, + toolDefinitions: input.hostTools?.toolDefinitions, + }, + ], + }, + }); + run.Graph!.overrideTestModel(['tool'], 5, [ + { id: 'large', name: 'inspect', args: {}, type: 'tool_call' }, + ]); + const original = run.processStream.bind(run); + run.processStream = async (...args) => { + const content = await original(...args); + workerContent = content; + workerMessages = run.getRunMessages(); + return content; + }; + workerRun = run; + return run; + }, + { maxFrameBytes: 20000, onError: (error) => diagnostics.push(String(error)) }, + ); + try { + const config = hostConfig(); + config.graphConfig = { + type: 'standard', + llmConfig: { provider: Providers.OPENAI }, + toolDefinitions: [{ name: 'inspect' }], + }; + config.customHandlers = Object.fromEntries( + Object.values(GraphEvents).map((event) => [ + event, + { + handle: async (_event: string, data: unknown) => { + largestEvent = Math.max(largestEvent, Buffer.byteLength(JSON.stringify(data))); + }, + }, + ]), + ); + config.customHandlers[GraphEvents.ON_TOOL_EXECUTE] = { + handle: async (_event, data) => { + effects++; + const batch = data as import('@librechat/agents').ToolExecuteBatchRequest; + batch.resolve([{ toolCallId: 'large', status: 'success', content: 'x'.repeat(12000) }]); + }, + }; + const run = await createRemoteSdkRun({ + config, + identity, + remote: { ...runtime.remote, maxFrameBytes: 20000 }, + }); + await expect( + run.processStream({ messages: [new HumanMessage('go')] }, invocation()), + ).resolves.toBeDefined(); + expect(effects).toBe(1); + expect(largestEvent).toBeLessThan(20000); + expect( + JSON.stringify({ content: workerContent, messages: workerMessages }).length, + ).toBeGreaterThan(20000); + expect(workerRun).toBeDefined(); + } finally { + await runtime.close(); + } +}); + +it('exposes the same active run steps through native inspection and streamed events', async () => { + let start!: () => void; + const started = new Promise((resolve) => { + start = resolve; + }); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const seen: unknown[] = []; + const runtime = await fixture(async (input) => { + const run = await Run.create({ + ...input.config, + returnContent: true, + tokenCounter: () => 1, + graphConfig: { + type: 'standard', + agents: [ + { + agentId: input.identity.agentId, + provider: Providers.OPENAI, + tools: [], + toolDefinitions: input.hostTools?.toolDefinitions, + }, + ], + }, + }); + run.Graph!.overrideTestModel(['tool'], 5, [ + { id: 'held', name: 'inspect', args: {}, type: 'tool_call' }, + ]); + return run; + }); + try { + const config = hostConfig(); + config.graphConfig = { + type: 'standard', + llmConfig: { provider: Providers.OPENAI }, + toolDefinitions: [{ name: 'inspect' }], + }; + config.customHandlers = { + [GraphEvents.ON_RUN_STEP]: { + handle: async (_event, data) => { + seen.push(data); + }, + }, + [GraphEvents.ON_TOOL_EXECUTE]: { + handle: async (_event, data) => { + const batch = data as import('@librechat/agents').ToolExecuteBatchRequest; + start(); + await held; + batch.resolve([{ toolCallId: 'held', status: 'success', content: 'finished' }]); + }, + }, + }; + const run = await createRemoteSdkRun({ config, identity, remote: runtime.remote }); + const completion = run.processStream({ messages: [new HumanMessage('go')] }, invocation()); + await started; + expect(seen.length).toBeGreaterThan(0); + expect(run.Graph!.getRunSteps()).toEqual(seen); + expect(run.Graph!.contentData).toEqual(seen); + expect(run.Graph!.getContentParts()!.length).toBeGreaterThan(0); + release(); + await completion; + } finally { + release(); + await runtime.close(); + } +}); + +it('retains host execution ownership until cancelled host-tool cleanup actually drains', async () => { + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let started!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + const controller = new AbortController(); + const runtime = await fixture(async (input) => { + const run = await Run.create({ + ...input.config, + returnContent: true, + tokenCounter: () => 1, + graphConfig: { + type: 'standard', + agents: [ + { + agentId: input.identity.agentId, + provider: Providers.OPENAI, + tools: [], + toolDefinitions: input.hostTools?.toolDefinitions, + }, + ], + }, + }); + run.processStream = async () => { + await input.config.customHandlers![GraphEvents.ON_TOOL_EXECUTE].handle( + GraphEvents.ON_TOOL_EXECUTE, + { + toolCalls: [{ id: 'held', name: 'inspect', args: {} }], + agentId: identity.agentId, + signal: input.signal, + resolve: () => undefined, + reject: () => undefined, + }, + ); + return []; + }; + return run; + }); + let hostSignal: AbortSignal | undefined; + let settled = false; + try { + const config = hostConfig(); + config.graphConfig = { + type: 'standard', + llmConfig: { provider: Providers.OPENAI }, + toolDefinitions: [{ name: 'inspect' }], + }; + config.customHandlers = { + [GraphEvents.ON_TOOL_EXECUTE]: { + handle: async (_event, data) => { + const batch = data as import('@librechat/agents').ToolExecuteBatchRequest; + hostSignal = batch.signal; + started(); + await held; + batch.reject(new Error('cancelled')); + }, + }, + }; + const run = await createRemoteSdkRun({ config, identity, remote: runtime.remote }); + const completion = run + .processStream({ messages: [] }, { ...invocation(), signal: controller.signal }) + .catch((error: Error) => error) + .finally(() => { + settled = true; + }); + await ready; + controller.abort(); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(hostSignal?.aborted).toBe(true); + expect(settled).toBe(false); + release(); + expect(await completion).toMatchObject({ name: 'AbortError' }); + } finally { + release(); + await runtime.close(); + } +}); + +it('rejects an old host transport before constructing the worker graph', async () => { + const createRun = jest.fn(factory); + const runtime = await fixture(createRun); + const socket = new WebSocket(runtime.remote.url, { + headers: { Authorization: `Bearer ${signedToken()}` }, + }); + try { + await new Promise((resolve, reject) => { + socket.once('open', resolve); + socket.once('error', reject); + }); + const peer = createSdkPeer(socket, { + maxFrameBytes: 65536, + maxBufferedBytes: 131072, + timeoutMs: 2000, + onClose: () => undefined, + onFrame: async () => undefined, + onCall: async () => null, + }); + await expect( + peer.call('run', { + operation: 'start', + input: { messages: [] }, + options: { + configurable: { + thread_id: identity.conversationId, + run_id: identity.runId, + user_id: identity.userId, + }, + }, + handlers: [], + hooks: [], + checkpoint: false, + agent: { id: identity.agentId }, + }), + ).rejects.toThrow(); + expect(createRun).not.toHaveBeenCalled(); + } finally { + socket.terminate(); + await runtime.close(); + } +}); diff --git a/packages/api/src/agents/remoteSdk/client.ts b/packages/api/src/agents/remoteSdk/client.ts index a872ff84b3..944466eb02 100644 --- a/packages/api/src/agents/remoteSdk/client.ts +++ b/packages/api/src/agents/remoteSdk/client.ts @@ -11,7 +11,7 @@ import { BaseCheckpointSaver, TOOL_APPROVAL_EXECUTION_SCOPE_CONFIG_KEY, } from '@librechat/agents'; -import type { RunConfig, RunInterruptResult } from '@librechat/agents'; +import type { RunConfig, RunInterruptResult, RunStep, RunStepClosedEvent } from '@librechat/agents'; import type { RemoteSdkOptions, RemoteSdkSnapshot, @@ -25,6 +25,7 @@ import { hydrateResumeRunSteps, createContentIndexOffsetHandlers } from '../hitl import { connectRecoveringSdkSocket } from './recovery/client'; import { remoteSdkLimits, isRemoteSdkSecret } from './types'; import { createHostToolService } from './hostTools'; +import { createSnapshotReceiver } from './snapshot'; import { createSdkPeer } from './peer'; /** Construct a real local Run whose execution is delegated to one operator-trusted runtime. */ @@ -64,22 +65,45 @@ export async function createRemoteSdkRun({ if (resumeState) { projection.contentParts.push(...resumeState.content); hydrateResumeRunSteps(resumeState.steps, projection.stepMap, graph, resumeState.content); + for (const step of projection.stepMap.values()) { + if (!step) continue; + graph.contentIndexMap.set(step.id, graph.contentData.length); + graph.contentData.push(step); + } } const projectionHandlers = createContentIndexOffsetHandlers( Object.fromEntries( Object.values(GraphEvents).map((event) => [ event, { - handle: (_event: string, data: SdkEventData) => + handle: (_event: string, data: SdkEventData) => { projection.aggregateContent({ event, data: data as Parameters[0]['data'], - }), + }); + if (event === GraphEvents.ON_RUN_STEP) { + const step = projection.stepMap.get((data as RunStep).id); + if (step) { + const position = graph.contentIndexMap.get(step.id); + if (position === undefined) { + graph.contentIndexMap.set(step.id, graph.contentData.length); + graph.contentData.push(step); + } else graph.contentData[position] = step; + hydrateResumeRunSteps([step], undefined, graph); + } + } else if (event === GraphEvents.ON_RUN_STEP_CLOSED) { + const closed = data as RunStepClosedEvent; + const step = graph.getRunStep(closed.id); + if (step) Object.assign(step, { status: closed.status, closed_at: closed.closed_at }); + } + }, }, ]), ), resumeState?.content, ); + const createResumeState = graph.createRunStepResumeState.bind(graph); + graph.createRunStepResumeState = () => snapshot?.runSteps ?? createResumeState(); graph.getContentParts = () => snapshot?.content ?? projection.contentParts.filter((part) => part != null); run.getRunMessages = () => snapshot?.messages ?? []; @@ -184,8 +208,10 @@ export async function createRemoteSdkRun({ }) : await legacy(); const execution = new AbortController(); + const terminal = createSnapshotReceiver(limits.maxBufferedBytes); snapshot = undefined; let settled = false; + const hostInvocations = new Set>(); const result = new Promise((resolve, reject) => { const finish = (value?: RemoteSdkSnapshot) => { if (settled) return; @@ -224,12 +250,15 @@ export async function createRemoteSdkRun({ frame.metadata, graph, ); + } else if (frame.type === 'snapshot') { + terminal.accept(frame.data); } else if (frame.type === 'done') { - finish(frame.data as RemoteSdkSnapshot); + if (hostInvocations.size) throw new Error('Host tool execution has not drained'); + finish(terminal.finish(frame.data)); } else if (frame.type === 'error') finish(); else throw new Error('Unexpected SDK runtime frame'); }, - onCall: async (method, data) => { + onCall: async (method, data, _timeoutMs, signal) => { if (execution.signal.aborted || options.signal?.aborted) throw new Error('SDK runtime execution stopped'); if (method === 'tools') { @@ -249,7 +278,13 @@ export async function createRemoteSdkRun({ peer, signal: execution.signal, }); - return hostToolInvoke(data); + const invocation = hostToolInvoke(data, signal); + hostInvocations.add(invocation); + try { + return await invocation; + } finally { + hostInvocations.delete(invocation); + } } if (method === 'hook') { if (config.hooks == null) throw new Error('Host hooks are unavailable'); @@ -351,6 +386,7 @@ export async function createRemoteSdkRun({ const agent = 'agents' in config.graphConfig ? config.graphConfig.agents[0] : config.graphConfig; const start: RemoteSdkStart = { + transportVersion: 2, hostTools: descriptors.length ? descriptors : undefined, operation, input, @@ -386,9 +422,14 @@ export async function createRemoteSdkRun({ agent: { id: identity.agentId, name: agent?.name, instructions: agent?.instructions }, }; void (async () => { + const capabilities = (await peer.call('capabilities', null)) as { + transportVersion?: number; + hostTools?: number; + }; + if (capabilities?.transportVersion !== 2) + throw new Error('Worker transport capability unavailable'); if (descriptors.length) { - const capabilities = (await peer.call('capabilities', null)) as { hostTools?: number }; - if (capabilities?.hostTools !== 1) + if (capabilities?.hostTools !== 2) throw new Error('Worker host tool capability unavailable'); } await peer.call('run', start); @@ -399,6 +440,10 @@ export async function createRemoteSdkRun({ if (snapshot.runSteps) graph.restoreRunStepResumeState(snapshot.runSteps); return snapshot.content?.filter((part) => part != null); } finally { + execution.abort(); + // A failed/cancelled remote run may leave host cleanup running. Keep the host + // execution enrolled until those handlers drain, even if the wire has closed. + await Promise.allSettled([...hostInvocations]); socket.close(1000); if ('waitForClose' in socket) { let timeout: ReturnType | undefined; diff --git a/packages/api/src/agents/remoteSdk/hostTools.spec.ts b/packages/api/src/agents/remoteSdk/hostTools.spec.ts index cd89c79f3a..24e51a8aa1 100644 --- a/packages/api/src/agents/remoteSdk/hostTools.spec.ts +++ b/packages/api/src/agents/remoteSdk/hostTools.spec.ts @@ -1,5 +1,14 @@ -import type { HostToolRequest, HostToolSessionUpdate } from '@librechat/agents'; +import { once } from 'node:events'; +import WebSocket, { WebSocketServer } from 'ws'; +import { createHostToolBindings, GraphEvents } from '@librechat/agents'; +import type { + HostToolRequest, + HostToolSessionUpdate, + ToolExecuteBatchRequest, +} from '@librechat/agents'; +import type { AddressInfo } from 'node:net'; import { createHostToolService, createWorkerToolService } from './hostTools'; +import { createSdkPeer } from './peer'; const identity = { runtimeId: 'runtime', @@ -109,3 +118,124 @@ it('delivers refreshed host session state before propagating an execution failur ).rejects.toThrow('execution failed'); expect(updates[0].context.codeSessionContext?.session_id).toBe('refreshed'); }); + +it.each([true, false])( + 'propagates worker cancellation with a cooperative=%s handler and preserves sessions', + async (cooperative) => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await once(server, 'listening'); + const connected = once(server, 'connection'); + const socket = new WebSocket(`ws://127.0.0.1:${(server.address() as AddressInfo).port}`); + await once(socket, 'open'); + const [hostSocket] = (await connected) as [WebSocket]; + const limits = { + maxFrameBytes: 65536, + maxBufferedBytes: 131072, + timeoutMs: 2000, + onClose: () => undefined, + onFrame: async () => undefined, + }; + const hostRun = new AbortController(); + const workerCall = new AbortController(); + let begin!: () => void; + const started = new Promise((resolve) => { + begin = resolve; + }); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let hostSignal: AbortSignal | undefined; + let effects = 0; + let resolveFinished!: () => void; + const handlerFinished = new Promise((resolve) => { + resolveFinished = resolve; + }); + const host = createHostToolService({ + identity, + tools, + configurable: {}, + signal: hostRun.signal, + peer: { call: (...args) => hostPeer.call(...args) }, + handler: { + handle: async (_event, data) => { + const batch = data as ToolExecuteBatchRequest; + hostSignal = batch.signal; + batch.toolCalls[0].codeSessionContext = { session_id: 'provisioned' }; + begin(); + if (cooperative) { + await new Promise((resolve) => + batch.signal!.addEventListener('abort', () => resolve(), { once: true }), + ); + await held; + batch.toolCalls[0].runtimeSessionHint = 'cancelled-session'; + batch.reject(new Error('cancelled')); + } else { + await held; + if (!batch.signal?.aborted) effects++; + batch.onResult?.({ toolCallId: 'call', status: 'success', content: 'late' }); + batch.resolve([{ toolCallId: 'call', status: 'success', content: 'late' }]); + } + resolveFinished(); + }, + }, + }); + const hostPeer = createSdkPeer(hostSocket, { + ...limits, + onCall: (method, data, _timeout, signal) => + method === 'tools' ? host(data, signal) : Promise.resolve('alive'), + }); + const worker = createWorkerToolService({ call: (...args) => workerPeer.call(...args) }); + const workerPeer = createSdkPeer(socket, { + ...limits, + onCall: async (method, data) => + worker.accept(data, method === 'tool.session' ? 'session' : 'result'), + }); + const bindings = createHostToolBindings({ identity, tools, service: worker.service }); + const resolve = jest.fn(); + const reject = jest.fn(); + const onResult = jest.fn(); + const call = { id: 'call', name: 'inspect', args: {} }; + try { + const operation = bindings.customHandlers[GraphEvents.ON_TOOL_EXECUTE].handle( + GraphEvents.ON_TOOL_EXECUTE, + { + toolCalls: [call], + signal: workerCall.signal, + agentId: identity.agentId, + resolve, + reject, + onResult, + }, + ); + await started; + workerCall.abort(); + let settled = false; + void Promise.resolve(operation).then(() => { + settled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(settled).toBe(false); + expect(hostSignal?.aborted).toBe(true); + release(); + await operation; + expect(hostSignal?.aborted).toBe(true); + expect(hostRun.signal.aborted).toBe(false); + expect(resolve).not.toHaveBeenCalled(); + expect(reject).toHaveBeenCalledTimes(1); + expect(call).toMatchObject({ codeSessionContext: { session_id: 'provisioned' } }); + if (cooperative) expect(call).toMatchObject({ runtimeSessionHint: 'cancelled-session' }); + release(); + await handlerFinished; + expect(effects).toBe(0); + expect(onResult).not.toHaveBeenCalled(); + expect(socket.readyState).toBe(WebSocket.OPEN); + await expect(workerPeer.call('probe', null)).resolves.toBe('alive'); + } finally { + release(); + hostSocket.terminate(); + socket.terminate(); + await new Promise((resolve) => server.close(() => resolve())); + } + }, +); diff --git a/packages/api/src/agents/remoteSdk/hostTools.ts b/packages/api/src/agents/remoteSdk/hostTools.ts index c8d97b5fd4..1cd975d364 100644 --- a/packages/api/src/agents/remoteSdk/hostTools.ts +++ b/packages/api/src/agents/remoteSdk/hostTools.ts @@ -30,9 +30,12 @@ export function createHostToolService(options: { metadata?: Record; peer: { call: SdkRpc }; signal: AbortSignal; -}): (data: unknown) => Promise { +}): (data: unknown, invocationSignal?: AbortSignal) => Promise { const references = new Map(options.tools.map((t) => [t.definition.name, t.reference])); - return async (data: unknown): Promise => { + return async (data: unknown, invocationSignal?: AbortSignal): Promise => { + const signal = invocationSignal + ? AbortSignal.any([options.signal, invocationSignal]) + : options.signal; const envelope = z .object({ id: z.string().uuid(), @@ -83,7 +86,7 @@ export function createHostToolService(options: { throw new Error('Host tool capability mismatch'); ids.add(call.id); } - options.signal.throwIfAborted(); + signal.throwIfAborted(); const notifications: Promise[] = []; let active = true; let resolve!: (results: ToolExecuteResult[]) => void; @@ -94,6 +97,8 @@ export function createHostToolService(options: { }); // Install rejection handling before invoking a handler that may reject synchronously. void result.catch(() => undefined); + const abort = () => reject(new Error('Host tool invocation cancelled')); + signal.addEventListener('abort', abort, { once: true }); try { await options.handler.handle( GraphEvents.ON_TOOL_EXECUTE, @@ -104,13 +109,13 @@ export function createHostToolService(options: { configurable: options.configurable, metadata: options.metadata, callerCapabilityProjection: request.callerCapabilityProjection, - signal: options.signal, + signal, resolve, reject, ...(envelope.incremental ? { onResult: (value: ToolExecuteResult) => { - if (!active || options.signal.aborted) return; + if (!active || signal.aborted) return; if (!ids.has(value.toolCallId)) throw new Error('Invalid incremental tool receipt'); const notice = options.peer.call('tool.result', { @@ -126,13 +131,17 @@ export function createHostToolService(options: { options.metadata, options.graph, ); + // Handler settlement is the drain boundary: async cancellation cleanup may refresh + // sessions. Never publish a final receipt while that cleanup is still running. + signal.throwIfAborted(); const results = await result; active = false; await Promise.all(notifications); - options.signal.throwIfAborted(); + signal.throwIfAborted(); return results; } finally { active = false; + signal.removeEventListener('abort', abort); for (const { call } of request.calls) { notifications.push( options.peer.call('tool.session', { @@ -150,6 +159,7 @@ export function createHostToolService(options: { ); } await Promise.all(notifications); + signal.throwIfAborted(); } }; } @@ -158,20 +168,25 @@ export function createWorkerToolService(peer: { call: SdkRpc }): { service: HostToolService; accept: (data: unknown, kind?: 'result' | 'session') => null; } { - const pending = new Map(); + const pending = new Map(); return { service: { version: 1, invoke: async (request, signal, events) => { signal?.throwIfAborted(); const id = randomUUID(); - if (events) pending.set(id, events); + if (events) pending.set(id, { events, signal }); try { - return (await peer.call('tools', { - id, - request, - incremental: Boolean(events?.onResult), - })) as ToolExecuteResult[]; + return (await peer.call( + 'tools', + { + id, + request, + incremental: Boolean(events?.onResult), + }, + undefined, + signal, + )) as ToolExecuteResult[]; } finally { pending.delete(id); } @@ -199,13 +214,13 @@ export function createWorkerToolService(peer: { call: SdkRpc }): { .parse(data); const events = pending.get(value.id); if (!events) throw new Error('Unknown tool session invocation'); - events.onSessionUpdate(value.update as HostToolSessionUpdate); + events.events.onSessionUpdate(value.update as HostToolSessionUpdate); return null; } const value = toolResultSchema.parse(data); const handler = pending.get(value.id); if (!handler) throw new Error('Unknown tool result invocation'); - handler.onResult?.(value.result as ToolExecuteResult); + if (!handler.signal?.aborted) handler.events.onResult?.(value.result as ToolExecuteResult); return null; }, }; diff --git a/packages/api/src/agents/remoteSdk/peer.spec.ts b/packages/api/src/agents/remoteSdk/peer.spec.ts index d2c96c60b6..58a8ba7c59 100644 --- a/packages/api/src/agents/remoteSdk/peer.spec.ts +++ b/packages/api/src/agents/remoteSdk/peer.spec.ts @@ -35,6 +35,46 @@ async function pair() { } describe('remote SDK event and policy ordering', () => { + it('cancels a queued invocation before dispatch without waiting for the event queue', async () => { + const sockets = await pair(); + const blocked = deferred(); + const release = deferred(); + const cancelled = deferred(); + const handle = jest.fn(async () => null); + sockets.host.on('message', (raw) => { + if (JSON.parse(raw.toString()).type === 'cancel') cancelled.resolve(); + }); + createSdkPeer(sockets.host, { + ...limits, + onCall: handle, + onFrame: async () => { + blocked.resolve(); + await release.promise; + }, + }); + const worker = createSdkPeer(sockets.worker, { + ...limits, + onCall: async () => null, + onFrame: async () => undefined, + }); + try { + worker.send({ type: 'event', name: 'held', data: null }); + await blocked.promise; + const controller = new AbortController(); + const call = worker.call('tools', {}, undefined, controller.signal); + const rejected = call.catch((error: Error) => error); + controller.abort(); + await cancelled.promise; + release.resolve(); + expect(await rejected).toMatchObject({ message: 'SDK runtime connection failed' }); + expect(handle).not.toHaveBeenCalled(); + expect(sockets.worker.readyState).toBe(WebSocket.OPEN); + await expect(worker.call('probe', null)).resolves.toBeNull(); + } finally { + release.resolve(); + await sockets.close(); + } + }); it('applies preceding async events before a hook, then allows subsequent events', async () => { const sockets = await pair(); const release = deferred(); diff --git a/packages/api/src/agents/remoteSdk/peer.ts b/packages/api/src/agents/remoteSdk/peer.ts index ad00c3c792..0b209aa9e3 100644 --- a/packages/api/src/agents/remoteSdk/peer.ts +++ b/packages/api/src/agents/remoteSdk/peer.ts @@ -14,15 +14,24 @@ export interface SdkSocket { once(event: 'close' | 'error', listener: () => void): this; } -export type SdkRpc = (method: string, data: unknown, timeoutMs?: number) => Promise; +/** Cancellation requests remote cleanup; settlement waits for its reply (or the RPC deadline). */ +export type SdkRpc = ( + method: string, + data: unknown, + timeoutMs?: number, + signal?: AbortSignal, +) => Promise; export type SdkFrame = | { type: 'call'; id: string; method: string; data: unknown } + | { type: 'cancel'; id: string } | { type: 'result'; id: string; data: unknown; failed?: boolean } | { type: 'event'; name: string; data: unknown; metadata?: Record } + | { type: 'snapshot'; data: unknown } | { type: 'done'; data: unknown } | { type: 'error' }; const frameSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('cancel'), id: z.string().uuid() }), z.object({ type: z.literal('call'), id: z.string().uuid(), @@ -41,6 +50,7 @@ const frameSchema = z.discriminatedUnion('type', [ data: z.unknown(), metadata: z.record(z.string(), z.unknown()).optional(), }), + z.object({ type: z.literal('snapshot'), data: z.unknown() }), z.object({ type: z.literal('done'), data: z.unknown() }), z.object({ type: z.literal('error') }), ]); @@ -62,7 +72,7 @@ export function createSdkPeer( string, { resolve: (data: unknown) => void; reject: (error: Error) => void } >(); - const incoming = new Set(); + const incoming = new Map(); let queue = Promise.resolve(); let closed = false; let queuedBytes = 0; @@ -89,6 +99,8 @@ export function createSdkPeer( closed = true; for (const entry of pending.values()) entry.reject(failure()); pending.clear(); + for (const controller of incoming.values()) controller.abort(); + incoming.clear(); options.onClose(failure()); }; socket.on('close', close); @@ -107,6 +119,12 @@ export function createSdkPeer( socket.close(1008); return; } + // Cancel before dispatch as well as during execution, without waiting behind event/hooks. + // Unknown/settled IDs are harmless; this cannot cancel another connection's invocation. + if (frame.type === 'cancel') { + incoming.get(frame.id)?.abort(); + return; + } // RPC replies must bypass the event queue: a handler may itself be awaiting this reply. if (frame.type === 'result') { const entry = pending.get(frame.id); @@ -122,7 +140,8 @@ export function createSdkPeer( socket.close(1008); return; } - incoming.add(frame.id); + const controller = new AbortController(); + incoming.set(frame.id, controller); queuedBytes += bytes; if (queuedBytes > options.maxBufferedBytes) { close(); @@ -133,7 +152,8 @@ export function createSdkPeer( Promise.resolve() .then(() => { if (closed) throw failure(); - return options.onCall(frame.method, frame.data); + controller.signal.throwIfAborted(); + return options.onCall(frame.method, frame.data, undefined, controller.signal); }) .then( (data) => send({ type: 'result', id: frame.id, data }), @@ -174,27 +194,43 @@ export function createSdkPeer( socket.close(1011); }); }); - const call: SdkRpc = async (method, data, timeoutMs = options.timeoutMs) => { + const call: SdkRpc = async (method, data, timeoutMs = options.timeoutMs, signal) => { + signal?.throwIfAborted(); if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) throw failure(); const id = randomUUID(); return new Promise((resolve, reject) => { + const abort = () => { + try { + send({ type: 'cancel', id }); + } catch { + close(); + socket.close(1011); + } + }; + const cleanup = () => { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + }; const timer = setTimeout(() => { + cleanup(); pending.delete(id); reject(failure()); }, timeoutMs); timer.unref(); pending.set(id, { resolve: (value) => { - clearTimeout(timer); + cleanup(); resolve(value); }, reject: (error) => { - clearTimeout(timer); + cleanup(); reject(error); }, }); try { send({ type: 'call', id, method, data }); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); } catch (error) { pending.get(id)?.reject(error instanceof Error ? error : failure()); pending.delete(id); diff --git a/packages/api/src/agents/remoteSdk/recovery/client.ts b/packages/api/src/agents/remoteSdk/recovery/client.ts index e517043b3b..5faa00f36e 100644 --- a/packages/api/src/agents/remoteSdk/recovery/client.ts +++ b/packages/api/src/agents/remoteSdk/recovery/client.ts @@ -15,10 +15,29 @@ export async function connectRecoveringSdkSocket(options: { onError?: (error: unknown) => void; }): Promise { const { limits, recovery } = options; - if (Object.values(recovery).some((v) => !Number.isSafeInteger(v) || v <= 0)) + if ( + [recovery.maxDisconnectMs, recovery.retryDelayMs, recovery.maxJournalBytes].some( + (v) => !Number.isSafeInteger(v) || v <= 0, + ) + ) throw new Error('Invalid SDK recovery configuration'); if (recovery.maxDisconnectMs >= limits.rpcTimeoutMs) throw new Error('SDK recovery window must be shorter than the RPC timeout'); + for (const [owner, route] of Object.entries(recovery.routes ?? {})) { + const url = new URL(route); + if ( + !owner || + owner.length > 128 || + url.username || + url.password || + url.search || + url.hash || + (url.protocol !== 'wss:' && + !(url.protocol === 'ws:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))) + ) + throw new Error('Invalid SDK recovery owner route'); + } + let ownerId: string | undefined; const sessionId = randomUUID(); let connected = false; let connecting = false; @@ -82,9 +101,9 @@ export async function connectRecoveringSdkSocket(options: { const connect = async () => { if (socket.readyState !== WebSocket.OPEN || connecting) return; connecting = true; - const wire = new WebSocket(options.url, { + const wire = new WebSocket(ownerId ? recovery.routes![ownerId] : options.url, { headers: { - Authorization: `Bearer ${options.token({ sessionId, after: socket.cursor, resume: connected, generation: options.generation })}`, + Authorization: `Bearer ${options.token({ sessionId, after: socket.cursor, resume: connected, ownerId, generation: options.generation })}`, }, handshakeTimeout: Math.min(limits.connectTimeoutMs, Math.max(1, deadline - Date.now())), maxPayload: limits.maxFrameBytes + RECOVERY_HEADER_BYTES, @@ -95,57 +114,70 @@ export async function connectRecoveringSdkSocket(options: { const early: WebSocket.RawData[] = []; let earlyBytes = 0; try { - const ready = await new Promise<{ connection: string; received: number }>((yes, no) => { - const timer = setTimeout( - () => no(new Error('SDK recovery handshake timed out')), - Math.min(limits.connectTimeoutMs, Math.max(1, deadline - Date.now())), - ); - const cleanup = (keepMessage = false) => { - clearTimeout(timer); - if (!keepMessage) wire.removeListener('message', message); - wire.removeListener('close', closed); - wire.removeListener('error', closed); - }; - const closed = () => { - cleanup(); - no(new Error('SDK recovery handshake failed')); - }; - let readySeen = false; - const message = (raw: WebSocket.RawData) => { - if (readySeen) { - earlyBytes += Buffer.byteLength(raw.toString()); - if (earlyBytes > limits.maxBufferedBytes) wire.terminate(); - else early.push(raw); - return; - } - try { - const frame = z - .object({ - version: z.literal(1), - kind: z.literal('ready'), - sessionId: z.literal(sessionId), - connection: z.string().uuid(), - received: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), - }) - .parse(JSON.parse(raw.toString())); - readySeen = true; - cleanup(true); - handoff = () => wire.removeListener('message', message); - yes(frame); - } catch { + const ready = await new Promise<{ connection: string; received: number; ownerId?: string }>( + (yes, no) => { + const timer = setTimeout( + () => no(new Error('SDK recovery handshake timed out')), + Math.min(limits.connectTimeoutMs, Math.max(1, deadline - Date.now())), + ); + const cleanup = (keepMessage = false) => { + clearTimeout(timer); + if (!keepMessage) wire.removeListener('message', message); + wire.removeListener('close', closed); + wire.removeListener('error', closed); + }; + const closed = () => { cleanup(); - no(new Error('Invalid SDK recovery handshake')); - } - }; - wire.on('message', message); - wire.once('close', closed); - wire.once('error', closed); - }); + no(new Error('SDK recovery handshake failed')); + }; + let readySeen = false; + const message = (raw: WebSocket.RawData) => { + if (readySeen) { + earlyBytes += Buffer.byteLength(raw.toString()); + if (earlyBytes > limits.maxBufferedBytes) wire.terminate(); + else early.push(raw); + return; + } + try { + const frame = z + .object({ + version: z.literal(1), + kind: z.literal('ready'), + ownerId: z.string().min(1).max(128).optional(), + sessionId: z.literal(sessionId), + connection: z.string().uuid(), + received: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + }) + .parse(JSON.parse(raw.toString())); + readySeen = true; + cleanup(true); + handoff = () => wire.removeListener('message', message); + yes(frame); + } catch { + cleanup(); + no(new Error('Invalid SDK recovery handshake')); + } + }; + wire.on('message', message); + wire.once('close', closed); + wire.once('error', closed); + }, + ); if (wire.readyState !== WebSocket.OPEN) throw new Error('SDK recovery handshake closed'); if (socket.readyState !== WebSocket.OPEN) { wire.terminate(); return; } + if ( + (ownerId && ready.ownerId !== ownerId) || + (ready.ownerId && + !Object.prototype.hasOwnProperty.call(recovery.routes ?? {}, ready.ownerId)) || + (recovery.routes && !ready.ownerId) + ) { + failed(new Error('SDK recovery owner route unavailable')); + return; + } + ownerId = ready.ownerId; connected = true; options.signal?.removeEventListener('abort', initialAbort); socket.attach(wire, ready.connection, ready.received); diff --git a/packages/api/src/agents/remoteSdk/recovery/journal.cache_integration.spec.ts b/packages/api/src/agents/remoteSdk/recovery/journal.cache_integration.spec.ts index 037fc5b115..cc82168514 100644 --- a/packages/api/src/agents/remoteSdk/recovery/journal.cache_integration.spec.ts +++ b/packages/api/src/agents/remoteSdk/recovery/journal.cache_integration.spec.ts @@ -207,3 +207,22 @@ describe('persistent runtime journal', () => { expect(value(await journal.read(id, 0, 10))).toMatchObject({ state: 'completed', head: 1 }); }); }); + +it('uses EVALSHA and falls back only when Redis guarantees the script was not run', async () => { + const eval_ = jest.fn((script: string, keys: number, ...args: Array) => + redis.eval(script, keys, ...args), + ); + const evalsha = jest.fn((sha: string, keys: number, ...args: Array) => + redis.evalsha(sha, keys, ...args), + ); + evalsha.mockRejectedValueOnce(new Error('NOSCRIPT No matching script')); + const journal = createRedisRuntimeJournal({ eval: eval_, evalsha }, limits); + const id = session(); + const lease = value(await journal.create(id, randomUUID())); + value(await journal.renew(id, lease)); + expect(eval_).toHaveBeenCalledTimes(1); + expect(evalsha).toHaveBeenCalledTimes(2); + evalsha.mockRejectedValueOnce(new Error('connection lost after execution')); + await expect(journal.renew(id, lease)).rejects.toThrow('connection lost'); + expect(eval_).toHaveBeenCalledTimes(1); +}); diff --git a/packages/api/src/agents/remoteSdk/recovery/journal.ts b/packages/api/src/agents/remoteSdk/recovery/journal.ts index 8be1b1882a..61fd34f0d2 100644 --- a/packages/api/src/agents/remoteSdk/recovery/journal.ts +++ b/packages/api/src/agents/remoteSdk/recovery/journal.ts @@ -38,6 +38,7 @@ export interface RuntimeJournalPage { export interface RuntimeJournal { readonly leaseMs: number; readonly maxBatchFrames: number; + readonly maxBatchBytes?: number; /** Privileged lifecycle cleanup; callers must authenticate deletion authority before storage access. */ purge(session: RuntimeJournalSession): Promise>; create( diff --git a/packages/api/src/agents/remoteSdk/recovery/log.ts b/packages/api/src/agents/remoteSdk/recovery/log.ts index fd25afca09..83f56c2ade 100644 --- a/packages/api/src/agents/remoteSdk/recovery/log.ts +++ b/packages/api/src/agents/remoteSdk/recovery/log.ts @@ -18,30 +18,34 @@ export function createJournalRecoveryLog(options: { }): RecoveryLog { let head = options.head; const { journal, session, lease } = options; - return { - head: () => head, - append: async (id, payload) => { - if (!lease) throw new Error('SDK recovery journal is read only'); + const appendBatch = async (id: string, payloads: string[]) => { + if (!lease) throw new Error('SDK recovery journal is read only'); + let terminal: Exclude | undefined; + for (const payload of payloads) { const entry = JSON.parse(payload) as { kind: string; payload: string }; - const frame = - entry.kind === 'data' - ? (JSON.parse(entry.payload) as { - type?: string; - data?: { interrupt?: { $sdk?: string } }; - }) - : undefined; - let terminal: Exclude | undefined; - if (frame?.type === 'error') terminal = 'failed'; - if (frame?.type === 'done') { + if (entry.kind !== 'data') continue; + if (terminal) throw new Error('SDK frames follow a terminal receipt'); + const frame = JSON.parse(entry.payload) as { + type?: string; + data?: { interrupt?: { $sdk?: string } }; + }; + if (frame.type === 'error') terminal = 'failed'; + if (frame.type === 'done') terminal = frame.data?.interrupt != null && frame.data.interrupt.$sdk !== 'undefined' ? 'paused' : 'completed'; - } - const range = journalValue(await journal.append(session, lease, id, [payload], terminal)); - head = range.last; - return range.last; - }, + } + const range = journalValue(await journal.append(session, lease, id, payloads, terminal)); + head = range.last; + return range; + }; + return { + head: () => head, + maxBatchFrames: journal.maxBatchBytes ? journal.maxBatchFrames : 1, + maxBatchBytes: journal.maxBatchBytes, + appendBatch, + append: async (id, payload) => (await appendBatch(id, [payload])).last, read: async (after) => { const page = journalValue(await journal.read(session, after, journal.maxBatchFrames)); head = page.head; diff --git a/packages/api/src/agents/remoteSdk/recovery/redis.ts b/packages/api/src/agents/remoteSdk/recovery/redis.ts index 5bb15c4b1b..220af0b09a 100644 --- a/packages/api/src/agents/remoteSdk/recovery/redis.ts +++ b/packages/api/src/agents/remoteSdk/recovery/redis.ts @@ -8,6 +8,7 @@ import type { /** Operator-supplied client; no app singleton or credentials cross the runtime boundary. */ export interface RuntimeJournalRedis { + evalsha?(sha: string, numberOfKeys: number, ...args: Array): Promise; eval(script: string, numberOfKeys: number, ...args: Array): Promise; } export interface RuntimeJournalLimits { @@ -116,6 +117,7 @@ if op == 'finish' then end return fail('closed') `; +const SCRIPT_SHA = createHash('sha1').update(SCRIPT).digest('hex'); const failures = new Set([ 'missing', 'scope_mismatch', @@ -166,9 +168,7 @@ export function createRedisRuntimeJournal( if (!/^[a-f0-9]{64}$/.test(session.scope)) throw new Error('Invalid runtime journal scope'); if (op !== 'read' && op !== 'purge') id(owner); const key = `${limits.prefix}:{${session.sessionId}}`; - const raw = await redis.eval( - SCRIPT, - 3, + const arguments_: Array = [ `${key}:meta`, `${key}:frames`, `${key}:receipts`, @@ -182,7 +182,17 @@ export function createRedisRuntimeJournal( limits.maxBytes, terminal ?? 0, ...args, - ); + ]; + let raw: unknown; + if (redis.evalsha) { + try { + raw = await redis.evalsha(SCRIPT_SHA, 3, ...arguments_); + } catch (error) { + // NOSCRIPT guarantees no execution. Never retry an ambiguous network failure. + if (!(error instanceof Error) || !/^NOSCRIPT\b/.test(error.message)) throw error; + raw = await redis.eval(SCRIPT, 3, ...arguments_); + } + } else raw = await redis.eval(SCRIPT, 3, ...arguments_); if (!Array.isArray(raw)) throw new Error('Invalid runtime journal response'); if (raw[0] === 0 && failures.has(raw[1])) return { ok: false, error: { code: raw[1] } }; if (raw[0] !== 1) throw new Error('Invalid runtime journal response'); @@ -191,6 +201,7 @@ export function createRedisRuntimeJournal( return { leaseMs: limits.leaseMs, maxBatchFrames: limits.maxBatchFrames, + maxBatchBytes: limits.maxBatchBytes, purge: (session) => invoke('purge', session, '', 0, [], () => null), create: (session, owner) => invoke('create', session, owner, 0, [], (r) => ({ owner, epoch: integer(r[1]) })), diff --git a/packages/api/src/agents/remoteSdk/recovery/runtime.cache_integration.spec.ts b/packages/api/src/agents/remoteSdk/recovery/runtime.cache_integration.spec.ts index e8d7d51055..4a81658f39 100644 --- a/packages/api/src/agents/remoteSdk/recovery/runtime.cache_integration.spec.ts +++ b/packages/api/src/agents/remoteSdk/recovery/runtime.cache_integration.spec.ts @@ -13,8 +13,8 @@ import { MemorySaver, HookRegistry, } from '@librechat/agents'; +import type { RunConfig, ToolCallRequest, ToolExecuteBatchRequest } from '@librechat/agents'; import type { AddressInfo, Socket } from 'node:net'; -import type { RunConfig } from '@librechat/agents'; import type { RedisTestClient } from '../../../stream/__tests__/helpers/redis'; import type { RemoteSdkFactoryInput } from '../types'; import { @@ -91,7 +91,7 @@ function hostConfig(handle: () => Promise) { }, }; } -async function fixture(makeRun = factory) { +async function fixture(makeRun = factory, ownerId?: string) { const createRun = jest.fn(makeRun); const errors: unknown[] = []; const journalPrefix = `${prefix}:${randomUUID()}`; @@ -116,7 +116,7 @@ async function fixture(makeRun = factory) { secret, createRun, authorize: () => true, - recovery: { journal, maxDisconnectMs: 5000 }, + recovery: { journal, maxDisconnectMs: 5000, ownerId }, onError: (error) => errors.push(error), }); let runtime = makeRuntime(); @@ -170,7 +170,12 @@ async function fixture(makeRun = factory) { remote: { url: `ws://127.0.0.1:${(proxy.address() as AddressInfo).port}/sdk`, secret, - recovery: { maxDisconnectMs: 5000, retryDelayMs: 20, maxJournalBytes: 16777216 }, + recovery: { + maxDisconnectMs: 5000, + retryDelayMs: 20, + maxJournalBytes: 16777216, + ...(ownerId ? { routes: { [ownerId]: `ws://127.0.0.1:${port}/sdk` } } : {}), + }, }, close: async () => { for (const socket of sockets) socket.destroy(); @@ -180,6 +185,104 @@ async function fixture(makeRun = factory) { }; } describe('live SDK recovery', () => { + it('replays worker tool cancellation across a disconnect without repeating the invocation', async () => { + const workerCall = new AbortController(); + let started!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + const resolveTool = jest.fn(); + const rejectTool = jest.fn(); + const toolCall: ToolCallRequest = { id: 'cancelled-call', name: 'inspect', args: {} }; + const runtime = await fixture(async (input) => { + const run = await Run.create({ + ...input.config, + tokenCounter: () => 1, + returnContent: true, + graphConfig: { + type: 'standard', + agents: [ + { + agentId: input.identity.agentId, + provider: Providers.OPENAI, + tools: [], + toolDefinitions: input.hostTools?.toolDefinitions, + }, + ], + }, + }); + run.processStream = async () => { + await input.config.customHandlers![GraphEvents.ON_TOOL_EXECUTE].handle( + GraphEvents.ON_TOOL_EXECUTE, + { + toolCalls: [toolCall], + signal: workerCall.signal, + agentId: identity.agentId, + resolve: resolveTool, + reject: rejectTool, + }, + ); + return []; + }; + return run; + }); + let calls = 0; + let hostSignal: AbortSignal | undefined; + try { + const config: RunConfig = { + ...hostConfig(async () => undefined), + graphConfig: { + type: 'standard', + llmConfig: { provider: Providers.OPENAI }, + toolDefinitions: [{ name: 'inspect', description: 'Inspect' }], + }, + customHandlers: { + [GraphEvents.ON_TOOL_EXECUTE]: { + handle: async (_event, data) => { + const batch = data as ToolExecuteBatchRequest; + calls++; + hostSignal = batch.signal; + batch.toolCalls[0].codeSessionContext = { session_id: 'retained-session' }; + started(); + await new Promise((resolve) => + batch.signal!.addEventListener('abort', () => resolve(), { once: true }), + ); + batch.reject(new Error('cancelled')); + }, + }, + }, + }; + const run = await createRemoteSdkRun({ config, identity, remote: runtime.remote }); + const completion = run.processStream( + { messages: [] }, + { + version: 'v2', + configurable: { thread_id: identity.conversationId }, + }, + ); + void completion.catch(() => undefined); + await ready; + runtime.offline(true); + runtime.drop(); + await delay(50); + workerCall.abort(); + await delay(50); + expect(hostSignal?.aborted).toBe(false); + runtime.offline(false); + await completion; + expect(hostSignal?.aborted).toBe(true); + expect(calls).toBe(1); + expect(runtime.createRun).toHaveBeenCalledTimes(1); + expect(runtime.connections()).toBeGreaterThan(1); + expect(resolveTool).not.toHaveBeenCalled(); + expect(rejectTool).toHaveBeenCalledTimes(1); + expect(toolCall.codeSessionContext?.session_id).toBe('retained-session'); + } finally { + runtime.offline(false); + workerCall.abort(); + await runtime.close(); + } + }); it('runs a real SDK graph through the negotiated durable channel', async () => { const runtime = await fixture(); try { @@ -559,3 +662,130 @@ describe('live SDK recovery', () => { }, ); }); + +it('reconnects directly to the allowlisted owner when the front door is unavailable', async () => { + const runtime = await fixture(factory, 'worker-a'); + let dropped = false; + try { + const run = await createRemoteSdkRun({ + identity, + remote: runtime.remote, + config: hostConfig(async () => { + if (dropped) return; + dropped = true; + runtime.offline(true); + runtime.drop(); + }), + }); + const result = await run.processStream( + { messages: [new HumanMessage('go')] }, + { version: 'v2', configurable: { thread_id: identity.conversationId } }, + ); + expect(JSON.stringify(result)).toContain(text); + expect(runtime.createRun).toHaveBeenCalledTimes(1); + expect(runtime.connections()).toBe(1); + expect(runtime.errors).toEqual([]); + } finally { + await runtime.close(); + } +}); + +it('serializes terminal replay admission and cannot evict a replacement session', async () => { + const journal = createRedisRuntimeJournal(redis, { + prefix, + retentionMs: 60000, + leaseMs: 3000, + maxFrames: 100, + maxBytes: 100000, + maxBatchFrames: 10, + maxBatchBytes: 10000, + }); + const recovery = createRuntimeRecovery({ + journal, + maxDisconnectMs: 5000, + maxFrameBytes: 10000, + maxBufferedBytes: 20000, + ownerId: 'worker-a', + }); + const claim = { sessionId: randomUUID(), after: 0, resume: false }; + try { + const first = await recovery.prepare(identity, 'thread', claim); + await first.begin(); + await first.complete('completed'); + first.dispose(); + const [a, b] = await Promise.all([ + recovery.prepare(identity, 'thread', { ...claim, resume: true }), + recovery.prepare(identity, 'thread', { ...claim, resume: true }), + ]); + expect(a).toBe(b); + await expect( + recovery.prepare({ ...identity, userId: 'foreign' }, 'thread', claim), + ).rejects.toThrow('identity'); + await expect( + recovery.prepare(identity, 'thread', { ...claim, ownerId: 'worker-b' }), + ).rejects.toThrow('owner'); + a.dispose(); + const replacement = await recovery.prepare(identity, 'thread', { ...claim, resume: true }); + a.dispose(); + expect(await recovery.prepare(identity, 'thread', claim)).toBe(replacement); + recovery.close(); + await expect(recovery.prepare(identity, 'thread', claim)).rejects.toThrow('closed'); + } finally { + recovery.close(); + } +}); + +it('rejects unallowlisted owner identities before invoking a graph', async () => { + const runtime = await fixture(factory, 'worker-a'); + try { + const run = await createRemoteSdkRun({ + identity, + config: hostConfig(async () => undefined), + remote: { ...runtime.remote, recovery: { ...runtime.remote.recovery, routes: {} } }, + }); + await expect( + run.processStream( + { messages: [] }, + { version: 'v2', configurable: { thread_id: identity.conversationId } }, + ), + ).rejects.toThrow(); + expect(runtime.createRun).not.toHaveBeenCalled(); + } finally { + await runtime.close(); + } +}); + +it('does not publish a session admitted concurrently with shutdown', async () => { + const journal = createRedisRuntimeJournal(redis, { + prefix, + retentionMs: 60000, + leaseMs: 3000, + maxFrames: 100, + maxBytes: 100000, + maxBatchFrames: 10, + maxBatchBytes: 10000, + }); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const read = journal.read.bind(journal); + journal.read = async (...args) => { + await held; + return read(...args); + }; + const recovery = createRuntimeRecovery({ + journal, + maxDisconnectMs: 5000, + maxFrameBytes: 10000, + maxBufferedBytes: 20000, + }); + const admission = recovery.prepare(identity, 'thread', { + sessionId: randomUUID(), + after: 0, + resume: false, + }); + recovery.close(); + release(); + await expect(admission).rejects.toThrow('closed'); +}); diff --git a/packages/api/src/agents/remoteSdk/recovery/server.ts b/packages/api/src/agents/remoteSdk/recovery/server.ts index 91f68ad120..5e5e6bb347 100644 --- a/packages/api/src/agents/remoteSdk/recovery/server.ts +++ b/packages/api/src/agents/remoteSdk/recovery/server.ts @@ -12,6 +12,7 @@ import { createJournalRecoveryLog, journalValue } from './log'; import { RecoverySocket } from './socket'; export interface RecoveryClaim { + ownerId?: string; sessionId: string; after: number; resume: boolean; @@ -19,6 +20,7 @@ export interface RecoveryClaim { } export const recoveryClaimSchema: z.ZodType = z .object({ + ownerId: z.string().min(1).max(128).optional(), sessionId: z.string().uuid(), after: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), resume: z.boolean(), @@ -26,6 +28,8 @@ export const recoveryClaimSchema: z.ZodType = z }) .strict(); export interface RuntimeRecoveryOptions { + /** Stable operator identity used for direct reconnect routing. */ + ownerId?: string; journal: RuntimeJournal; maxDisconnectMs?: number; } @@ -54,7 +58,11 @@ export function createRuntimeRecovery( close(): void; } { const sessions = new Map(); + const pending = new Map>(); + let closed = false; const graceMs = options.maxDisconnectMs ?? 15000; + if (options.ownerId != null && (!options.ownerId.length || options.ownerId.length > 128)) + throw new Error('Invalid SDK recovery owner identity'); if (!Number.isSafeInteger(graceMs) || graceMs <= 0 || options.journal.leaseMs < 30) throw new Error('Invalid SDK recovery settings'); const report = (error: unknown) => { @@ -64,158 +72,178 @@ export function createRuntimeRecovery( /* Trusted diagnostics only. */ } }; - return { - async prepare( - identity: Readonly, - threadId: string, - claim: RecoveryClaim, - ): Promise { - const scope = createHash('sha256') - .update( - JSON.stringify([ - identity.runtimeId, - identity.agentId, - identity.runId, - identity.userId, - identity.tenantId ?? null, - identity.conversationId ?? null, - threadId, - claim.generation ?? null, - ]), - ) - .digest('hex'); - const existing = sessions.get(claim.sessionId); - if (existing) { - if (existing.scope !== scope || existing.value.socket.readyState !== 1) - throw new Error('SDK recovery identity mismatch'); - return existing.value; - } - const session: RuntimeJournalSession = { sessionId: claim.sessionId, scope }; - const journal = options.journal; - let lease: RuntimeJournalLease | undefined; - let page = await journal.read(session, 0, 1); - let needsEngine = false; - if (!page.ok) { - if (page.error.code !== 'missing' || claim.resume) - throw new Error('SDK recovery session unavailable'); - lease = journalValue(await journal.create(session, randomUUID())); - page = await journal.read(session, 0, 1); - needsEngine = true; - } else if (page.value.state === 'open') { - const claimed = journalValue(await journal.claim(session, randomUUID())); - lease = claimed; - needsEngine = !claimed.started; - } - const initial = journalValue(page); - const log = createJournalRecoveryLog({ journal, session, lease, head: initial.head }); - if (!needsEngine && initial.state === 'open') { - // The previous process crossed the execution-start barrier. Its effects are uncertain. - // Replay committed output and fail; never invoke its factory again. - await log.append( - randomUUID(), - JSON.stringify({ kind: 'data', payload: '{"type":"error"}' }), - ); - } - const controller = new AbortController(); - let deadline: ReturnType | undefined; - let heartbeat: ReturnType | undefined; - let renewing = false; - let running = false; - let settled = false; - const expire = () => { - clearTimeout(deadline); - deadline = setTimeout(() => socket.terminate(), graceMs); - deadline.unref(); - }; - const socket = new RecoverySocket({ - log, - maxFrameBytes: options.maxFrameBytes, - maxBufferedBytes: options.maxBufferedBytes, - terminalFailure: initial.state === 'failed' || initial.state === 'cancelled', - onDisconnect: expire, - onFailure: (error) => { - report(error); - controller.abort(); - }, - }); - const dispose = () => { - clearTimeout(deadline); + const prepare = async ( + identity: Readonly, + threadId: string, + claim: RecoveryClaim, + ): Promise => { + if (closed) throw new Error('SDK recovery closed'); + if (claim.ownerId != null && claim.ownerId !== options.ownerId) + throw new Error('SDK recovery owner mismatch'); + const scope = createHash('sha256') + .update( + JSON.stringify([ + identity.runtimeId, + identity.agentId, + identity.runId, + identity.userId, + identity.tenantId ?? null, + identity.conversationId ?? null, + threadId, + claim.generation ?? null, + ]), + ) + .digest('hex'); + const existing = sessions.get(claim.sessionId); + if (existing) { + if (existing.scope !== scope || existing.value.socket.readyState !== 1) + throw new Error('SDK recovery identity mismatch'); + return existing.value; + } + const session: RuntimeJournalSession = { sessionId: claim.sessionId, scope }; + const journal = options.journal; + let lease: RuntimeJournalLease | undefined; + let page = await journal.read(session, 0, 1); + let needsEngine = false; + if (!page.ok) { + if (page.error.code !== 'missing' || claim.resume) + throw new Error('SDK recovery session unavailable'); + lease = journalValue(await journal.create(session, randomUUID())); + page = await journal.read(session, 0, 1); + needsEngine = true; + } else if (page.value.state === 'open') { + const claimed = journalValue(await journal.claim(session, randomUUID())); + lease = claimed; + needsEngine = !claimed.started; + } + const initial = journalValue(page); + const log = createJournalRecoveryLog({ journal, session, lease, head: initial.head }); + if (!needsEngine && initial.state === 'open') { + // The previous process crossed the execution-start barrier. Its effects are uncertain. + // Replay committed output and fail; never invoke its factory again. + await log.append(randomUUID(), JSON.stringify({ kind: 'data', payload: '{"type":"error"}' })); + } + const controller = new AbortController(); + let deadline: ReturnType | undefined; + let heartbeat: ReturnType | undefined; + let renewing = false; + let running = false; + let settled = false; + const expire = () => { + clearTimeout(deadline); + deadline = setTimeout(() => socket.terminate(), graceMs); + deadline.unref(); + }; + const socket = new RecoverySocket({ + log, + maxFrameBytes: options.maxFrameBytes, + maxBufferedBytes: options.maxBufferedBytes, + terminalFailure: initial.state === 'failed' || initial.state === 'cancelled', + onDisconnect: expire, + onFailure: (error) => { + report(error); + controller.abort(); + }, + }); + const dispose = () => { + clearTimeout(deadline); + clearInterval(heartbeat); + controller.abort(); + socket.terminate(); + if (sessions.get(claim.sessionId)?.value === value) sessions.delete(claim.sessionId); + }; + const value: RuntimeRecoverySession = { + socket, + controller, + needsEngine, + begin: async () => { + if (!lease || !needsEngine) throw new Error('SDK recovery execution is already started'); + const marker = journalValue(await journal.markStarted(session, lease)); + if (!marker.first) throw new Error('SDK recovery execution is already started'); + running = true; + }, + complete: async (state) => { + if (settled) return; + settled = true; + running = false; + await socket.flush(); + if (lease && journalValue(await journal.read(session, 0, 1)).state === 'open') + journalValue(await journal.finish(session, lease, state, log.head())); clearInterval(heartbeat); - controller.abort(); - socket.terminate(); - sessions.delete(claim.sessionId); - }; - const value: RuntimeRecoverySession = { - socket, - controller, - needsEngine, - begin: async () => { - if (!lease || !needsEngine) throw new Error('SDK recovery execution is already started'); - const marker = journalValue(await journal.markStarted(session, lease)); - if (!marker.first) throw new Error('SDK recovery execution is already started'); - running = true; - }, - complete: async (state) => { - if (settled) return; - settled = true; - running = false; - await socket.flush(); - if (lease && journalValue(await journal.read(session, 0, 1)).state === 'open') - journalValue(await journal.finish(session, lease, state, log.head())); - clearInterval(heartbeat); - if (socket.readyState !== 1) dispose(); - else expire(); - }, - attach: (wire, after) => { - clearTimeout(deadline); - const connection = randomUUID(); - socket.attach( - wire, + if (socket.readyState !== 1) dispose(); + else expire(); + }, + attach: (wire, after) => { + clearTimeout(deadline); + const connection = randomUUID(); + socket.attach( + wire, + connection, + after, + JSON.stringify({ + version: 1, + kind: 'ready', + ownerId: options.ownerId, connection, - after, - JSON.stringify({ - version: 1, - kind: 'ready', - connection, - sessionId: claim.sessionId, - received: socket.cursor, - }), - ); - if (settled || !needsEngine) expire(); - }, - dispose, - }; - socket.once('close', () => { - controller.abort(); - if (!running) dispose(); - }); - if (lease && needsEngine) { - heartbeat = setInterval( - () => { - if (renewing) return; - renewing = true; - void journal - .renew(session, lease!) - .then(journalValue) - .catch((error) => { - if (!settled) { - report(error); - dispose(); - } - }) - .finally(() => { - renewing = false; - }); - }, - Math.floor(journal.leaseMs / 3), + sessionId: claim.sessionId, + received: socket.cursor, + }), ); - heartbeat.unref(); + if (settled || !needsEngine) expire(); + }, + dispose, + }; + socket.once('close', () => { + controller.abort(); + if (!running) dispose(); + }); + if (lease && needsEngine) { + heartbeat = setInterval( + () => { + if (renewing) return; + renewing = true; + void journal + .renew(session, lease!) + .then(journalValue) + .catch((error) => { + if (!settled) { + report(error); + dispose(); + } + }) + .finally(() => { + renewing = false; + }); + }, + Math.floor(journal.leaseMs / 3), + ); + heartbeat.unref(); + } + if (closed) { + dispose(); + throw new Error('SDK recovery closed'); + } + sessions.set(claim.sessionId, { scope, value }); + expire(); + return value; + }; + return { + async prepare(identity, threadId, claim) { + // Serialize admission by session, then recheck the caller's full scope. + const previous = pending.get(claim.sessionId); + const operation = (async () => { + if (previous) await previous.catch(() => undefined); + return prepare(identity, threadId, claim); + })(); + pending.set(claim.sessionId, operation); + try { + return await operation; + } finally { + if (pending.get(claim.sessionId) === operation) pending.delete(claim.sessionId); } - sessions.set(claim.sessionId, { scope, value }); - expire(); - return value; }, close: () => { + closed = true; for (const entry of [...sessions.values()]) entry.value.dispose(); }, }; diff --git a/packages/api/src/agents/remoteSdk/recovery/socket.spec.ts b/packages/api/src/agents/remoteSdk/recovery/socket.spec.ts new file mode 100644 index 0000000000..6d1c3437f3 --- /dev/null +++ b/packages/api/src/agents/remoteSdk/recovery/socket.spec.ts @@ -0,0 +1,121 @@ +import { randomUUID } from 'node:crypto'; +import { EventEmitter } from 'node:events'; +import type WebSocket from 'ws'; +import type { RecoveryLog } from './socket'; +import { RecoverySocket } from './socket'; + +function fixture(overrides: Partial = {}) { + const entries: string[] = []; + const appendBatch = jest.fn(async (_id: string, payloads: string[]) => { + const first = entries.length + 1; + entries.push(...payloads); + return { first, last: entries.length }; + }); + const log: RecoveryLog = { + head: () => entries.length, + append: async (id, payload) => (await appendBatch(id, [payload])).last, + appendBatch, + maxBatchFrames: 8, + maxBatchBytes: 1024, + read: async (after) => + entries.slice(after).map((payload, i) => ({ sequence: after + i + 1, payload })), + acknowledge: async () => undefined, + ...overrides, + }; + const onFailure = jest.fn(); + const socket = new RecoverySocket({ + log, + maxFrameBytes: 256, + maxBufferedBytes: 2048, + onDisconnect: () => undefined, + onFailure, + }); + const wire = Object.assign(new EventEmitter(), { + readyState: 1, + bufferedAmount: 0, + send: jest.fn(), + close: jest.fn(), + terminate: jest.fn(), + }); + const connection = randomUUID(); + socket.attach(wire as unknown as WebSocket, connection, 0); + const receive = (kind: string, sequence: number, payload = '') => + wire.emit( + 'message', + Buffer.from(JSON.stringify({ version: 1, kind, connection, sequence }) + '\n' + payload), + ); + return { socket, wire, receive, appendBatch, entries, onFailure }; +} + +it('batches a burst within configured journal bounds and publishes in sequence', async () => { + const f = fixture(); + try { + await f.socket.flush(); + for (let i = 0; i < 32; i++) f.socket.send(String(i)); + await f.socket.flush(); + expect(f.appendBatch).toHaveBeenCalledTimes(4); + expect(f.entries.map((s) => JSON.parse(s).payload)).toEqual( + Array.from({ length: 32 }, (_, i) => String(i)), + ); + expect(f.wire.send.mock.calls.map(([s]) => JSON.parse(s.split('\n')[0]).sequence)).toEqual( + Array.from({ length: 32 }, (_, i) => i + 1), + ); + expect(f.socket.bufferedAmount).toBe(0); + expect(f.onFailure).not.toHaveBeenCalled(); + } finally { + f.socket.terminate(); + } +}); + +it('coalesces held durable ACKs without queuing raw frames or blocking cancellation', async () => { + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const acknowledge = jest.fn(async () => held); + const f = fixture({ acknowledge, head: () => 10000 }); + const receive = jest.fn(); + f.socket.on('message', receive); + try { + await f.socket.flush(); + f.receive('ack', 1); + await Promise.resolve(); + for (let sequence = 2; sequence <= 10000; sequence++) f.receive('ack', sequence); + f.receive('data', 1, '{"type":"cancel","id":"call"}'); + expect(receive).toHaveBeenCalledTimes(1); + expect(acknowledge).toHaveBeenCalledTimes(1); + expect(f.socket.bufferedAmount).toBe(0); + release(); + await new Promise((resolve) => setImmediate(resolve)); + expect(acknowledge).toHaveBeenLastCalledWith(10000); + expect(acknowledge).toHaveBeenCalledTimes(2); + expect(f.onFailure).not.toHaveBeenCalled(); + } finally { + release(); + f.socket.terminate(); + } +}); + +it('rejects an outgoing burst before exceeding the memory budget', async () => { + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const f = fixture({ + appendBatch: async () => { + await held; + return { first: 1, last: 1 }; + }, + }); + try { + await f.socket.flush(); + expect(() => { + for (let i = 0; i < 100; i++) f.socket.send('x'.repeat(200)); + }).toThrow('capacity'); + expect(f.socket.bufferedAmount).toBeLessThanOrEqual(2048); + } finally { + release(); + f.socket.terminate(); + await f.socket.flush(); + } +}); diff --git a/packages/api/src/agents/remoteSdk/recovery/socket.ts b/packages/api/src/agents/remoteSdk/recovery/socket.ts index 7711066f94..29da6c4f92 100644 --- a/packages/api/src/agents/remoteSdk/recovery/socket.ts +++ b/packages/api/src/agents/remoteSdk/recovery/socket.ts @@ -34,6 +34,9 @@ const header = z.discriminatedUnion('kind', [ ]); export interface RecoveryLog { append(id: string, payload: string): Promise; + appendBatch?(id: string, payloads: string[]): Promise<{ first: number; last: number }>; + maxBatchFrames?: number; + maxBatchBytes?: number; read(after: number): Promise>; acknowledge(sequence: number): Promise; head(): number; @@ -71,7 +74,16 @@ export class RecoverySocket extends EventEmitter { private wire?: WebSocket; private connection?: string; private tail = Promise.resolve(); - private incoming = Promise.resolve(); + private pendingFrames: Array<{ + kind: 'data' | 'end'; + payload: string; + encoded: string; + bytes: number; + }> = []; + + private scheduled = false; + private acknowledgement?: Promise; + private requestedAck = 0; private received = 0; private acknowledged = 0; private queuedBytes = 0; @@ -140,26 +152,76 @@ export class RecoverySocket extends EventEmitter { this.wire!.send(data); } - private queue(kind: 'data' | 'end', payload: string): void { - const bytes = Buffer.byteLength(payload); - if ( - bytes > this.options.maxFrameBytes || - this.bufferedAmount + bytes > this.options.maxBufferedBytes - ) { - throw new Error('SDK recovery frame capacity exceeded'); - } - this.queuedBytes += bytes; - const id = randomUUID(); + private scheduleFlush(): void { + if (this.scheduled || this.closed || !this.pendingFrames.length) return; + this.scheduled = true; this.tail = this.tail .then(async () => { - const sequence = await this.options.log.append(id, JSON.stringify({ kind, payload })); - if (kind === 'end') this.endSequence = sequence; - if (!this.replaying) this.publish(kind, sequence, payload); + const maxFrames = this.options.log.appendBatch ? (this.options.log.maxBatchFrames ?? 1) : 1; + const maxBytes = this.options.log.maxBatchBytes ?? this.options.maxBufferedBytes; + let count = 0; + let bytes = 0; + for (const frame of this.pendingFrames) { + if (count >= maxFrames || (count > 0 && bytes + frame.bytes > maxBytes)) break; + count++; + bytes += frame.bytes; + } + const batch = this.pendingFrames.splice(0, count); + if (!batch.length) return; + try { + const id = randomUUID(); + const range = this.options.log.appendBatch + ? await this.options.log.appendBatch( + id, + batch.map((f) => f.encoded), + ) + : { first: await this.options.log.append(id, batch[0].encoded) }; + for (let index = 0; index < batch.length; index++) { + const frame = batch[index]; + const sequence = range.first + index; + if (frame.kind === 'end') this.endSequence = sequence; + if (!this.replaying) this.publish(frame.kind, sequence, frame.payload); + } + } finally { + this.queuedBytes -= bytes; + } + }) + .catch(this.fail) + .finally(() => { + this.scheduled = false; + this.scheduleFlush(); + }); + } + + private queue(kind: 'data' | 'end', payload: string): void { + const encoded = JSON.stringify({ kind, payload }); + const bytes = Buffer.byteLength(encoded); + if ( + Buffer.byteLength(payload) > this.options.maxFrameBytes || + this.bufferedAmount + bytes > this.options.maxBufferedBytes + ) + throw new Error('SDK recovery frame capacity exceeded'); + this.queuedBytes += bytes; + this.pendingFrames.push({ kind, payload, encoded, bytes }); + this.scheduleFlush(); + } + + private persistAcknowledgement(): void { + if (this.acknowledgement || this.closed || this.requestedAck <= this.acknowledged) return; + this.acknowledgement = Promise.resolve() + .then(async () => { + const sequence = this.requestedAck; + await this.options.log.acknowledge(sequence); + this.acknowledged = sequence; + if (this.endSequence != null && sequence >= this.endSequence) this.terminate(); + }) + .catch((error) => { + if (!this.closed) this.fail(error); }) .finally(() => { - this.queuedBytes -= bytes; - }) - .catch(this.fail); + this.acknowledgement = undefined; + this.persistAcknowledgement(); + }); } send(data: string): void { @@ -181,6 +243,8 @@ export class RecoverySocket extends EventEmitter { terminate(): void { if (this.closed) return; this.closed = true; + for (const frame of this.pendingFrames) this.queuedBytes -= frame.bytes; + this.pendingFrames.length = 0; this.generation++; this.wire?.terminate(); this.wire = undefined; @@ -189,7 +253,9 @@ export class RecoverySocket extends EventEmitter { } async flush(): Promise { - await this.tail; + do { + await this.tail; + } while (!this.closed && (this.scheduled || this.pendingFrames.length)); if (this.failure) throw this.failure; } @@ -217,55 +283,55 @@ export class RecoverySocket extends EventEmitter { wire.once('error', detached); wire.on('message', (raw) => { if (generation !== this.generation || this.closed) return; - this.incoming = this.incoming - .then(async () => { - if (generation !== this.generation || this.closed) return; - const text = raw.toString(); - const split = text.indexOf('\n'); - if (split < 0 || Buffer.byteLength(text.slice(0, split)) > RECOVERY_HEADER_BYTES) - throw new Error('Invalid SDK recovery envelope'); - const frame = header.parse(JSON.parse(text.slice(0, split))); - const payload = text.slice(split + 1); - if ( - frame.connection !== this.connection || - Buffer.byteLength(payload) > this.options.maxFrameBytes - ) - throw new Error('Invalid SDK recovery frame'); - if (frame.kind === 'failure') { - if (frame.sequence !== this.received) - throw new Error('SDK recovery terminal cursor mismatch'); - this.emit('message', Buffer.from('{"type":"error"}')); - return; - } - if (frame.kind === 'ack') { - if (frame.sequence > this.options.log.head()) - throw new Error('SDK recovery cursor is ahead'); - if (frame.sequence > this.acknowledged) { - await this.options.log.acknowledge(frame.sequence); - this.acknowledged = frame.sequence; - } - if (this.endSequence != null && frame.sequence >= this.endSequence) this.terminate(); - return; - } - if (frame.sequence <= this.received) { - this.publish('ack', this.received); - return; - } - if (frame.sequence !== this.received + 1) throw new Error('SDK recovery sequence gap'); - this.received = frame.sequence; - if (frame.kind === 'end') { - this.publish('ack', this.received); - // Allow the acknowledgement to leave before closing the physical connection. - this.closed = true; - this.endResolve?.(); - this.emit('close'); - wire.close(1000); - } else { - this.emit('message', Buffer.from(payload)); - this.publish('ack', this.received); - } - }) - .catch(this.fail); + // No async receive queue: durable ACK bookkeeping must not retain raw frames or + // delay cancellation. The SDK peer bounds outstanding event/RPC handler work. + try { + const text = raw.toString(); + if (Buffer.byteLength(text) > this.options.maxBufferedBytes) + throw new Error('SDK recovery receive capacity exceeded'); + const split = text.indexOf('\n'); + if (split < 0 || Buffer.byteLength(text.slice(0, split)) > RECOVERY_HEADER_BYTES) + throw new Error('Invalid SDK recovery envelope'); + const frame = header.parse(JSON.parse(text.slice(0, split))); + const payload = text.slice(split + 1); + if ( + frame.connection !== this.connection || + Buffer.byteLength(payload) > this.options.maxFrameBytes + ) + throw new Error('Invalid SDK recovery frame'); + if (frame.kind === 'failure') { + if (frame.sequence !== this.received) + throw new Error('SDK recovery terminal cursor mismatch'); + this.emit('message', Buffer.from('{"type":"error"}')); + return; + } + if (frame.kind === 'ack') { + if (frame.sequence > this.options.log.head()) + throw new Error('SDK recovery cursor is ahead'); + this.requestedAck = Math.max(this.requestedAck, frame.sequence); + this.persistAcknowledgement(); + return; + } + if (frame.sequence <= this.received) { + this.publish('ack', this.received); + return; + } + if (frame.sequence !== this.received + 1) throw new Error('SDK recovery sequence gap'); + this.received = frame.sequence; + if (frame.kind === 'end') { + this.publish('ack', this.received); + // Allow the acknowledgement to leave before closing the physical connection. + this.closed = true; + this.endResolve?.(); + this.emit('close'); + wire.close(1000); + } else { + this.emit('message', Buffer.from(payload)); + this.publish('ack', this.received); + } + } catch (error) { + this.fail(error); + } }); if (ready) wire.send(ready); // Serialize replay with publication so new live frames cannot overtake its prefix. @@ -275,7 +341,7 @@ export class RecoverySocket extends EventEmitter { let cursor = after; while (generation === this.generation && this.hasWire) { const frames = await this.options.log.read(cursor); - if (!frames.length) break; + if (generation !== this.generation || !this.hasWire || !frames.length) break; for (const frame of frames) { const entry = z .object({ kind: z.enum(['data', 'end']), payload: z.string() }) diff --git a/packages/api/src/agents/remoteSdk/server.ts b/packages/api/src/agents/remoteSdk/server.ts index f306397aed..186605ec2c 100644 --- a/packages/api/src/agents/remoteSdk/server.ts +++ b/packages/api/src/agents/remoteSdk/server.ts @@ -20,6 +20,7 @@ import { remoteSdkLimits, isRemoteSdkSecret } from './types'; import { RECOVERY_HEADER_BYTES } from './recovery/socket'; import { createWorkerToolService } from './hostTools'; import { RemoteSdkCheckpointer } from './checkpoint'; +import { sendSnapshot } from './snapshot'; import { createSdkPeer } from './peer'; const identitySchema = z @@ -42,6 +43,7 @@ const claimSchema = z.object({ exp: z.number(), }); const startSchema = z.object({ + transportVersion: z.literal(2), hostTools: z .array( z.object({ @@ -249,8 +251,8 @@ export function createRemoteSdkServer(options: RemoteSdkServerOptions): { }; transport.once('close', cleanup); const workerTools = createWorkerToolService({ - call: (method, data) => - peer.call(method, data, method === 'tools' ? limits.maxRunMs : undefined), + call: (method, data, _timeoutMs, signal) => + peer.call(method, data, method === 'tools' ? limits.maxRunMs : undefined, signal), }); const peer = createSdkPeer(transport, { ...limits, @@ -261,7 +263,7 @@ export function createRemoteSdkServer(options: RemoteSdkServerOptions): { throw new Error('Unexpected runtime frame'); }, onCall: async (method, data) => { - if (method === 'capabilities') return { hostTools: 1 }; + if (method === 'capabilities') return { transportVersion: 2, hostTools: 2 }; if (method === 'tool.result') return workerTools.accept(data); if (method === 'tool.session') return workerTools.accept(data, 'session'); if (method !== 'run' || started) throw new Error('SDK execution already admitted'); @@ -383,9 +385,9 @@ export function createRemoteSdkServer(options: RemoteSdkServerOptions): { ) : await run.processStream(request.input, callerOptions); outcome = run.getInterrupt() ? 'paused' : 'completed'; - peer.send({ - type: 'done', - data: { + await sendSnapshot( + peer, + { content, messages: run.getRunMessages(), state: { @@ -403,7 +405,9 @@ export function createRemoteSdkServer(options: RemoteSdkServerOptions): { haltReason: run.getHaltReason(), runSteps: run.Graph?.createRunStepResumeState(), }, - }); + limits, + recovered ? () => recovered.socket.flush() : undefined, + ); }; running = true; void execute() diff --git a/packages/api/src/agents/remoteSdk/snapshot.spec.ts b/packages/api/src/agents/remoteSdk/snapshot.spec.ts new file mode 100644 index 0000000000..0b06301a0d --- /dev/null +++ b/packages/api/src/agents/remoteSdk/snapshot.spec.ts @@ -0,0 +1,38 @@ +import type { RemoteSdkSnapshot } from './types'; +import { createSnapshotReceiver, sendSnapshot } from './snapshot'; +import { remoteSdkLimits } from './types'; + +it('reassembles a large Unicode snapshot before accepting its terminal receipt', async () => { + const receiver = createSnapshotReceiver(100000); + const snapshot = { + content: [{ type: 'text', text: '🙂'.repeat(10000) }], + messages: [], + } as unknown as RemoteSdkSnapshot; + let result: RemoteSdkSnapshot | undefined; + let chunks = 0; + await sendSnapshot( + { + send: (frame) => { + expect(Buffer.byteLength(JSON.stringify(frame))).toBeLessThan(1024); + if (frame.type === 'snapshot') { + chunks++; + receiver.accept(frame.data); + } else result = receiver.finish(frame.data); + }, + }, + snapshot, + remoteSdkLimits({ maxFrameBytes: 1024, maxBufferedBytes: 100000 }), + ); + expect(chunks).toBeGreaterThan(1); + expect(result).toEqual(snapshot); + expect(() => receiver.finish({ snapshotBytes: 1 })).toThrow(); +}); + +it('rejects incomplete, reordered, and oversized transfers', () => { + const receiver = createSnapshotReceiver(100); + expect(() => receiver.accept({ offset: 0, total: 101, data: 'YQ==' })).toThrow(); + expect(() => receiver.accept({ offset: 1, total: 2, data: 'YQ==' })).toThrow(); + receiver.accept({ offset: 0, total: 2, data: 'YQ==' }); + expect(() => receiver.accept({ offset: 0, total: 2, data: 'YQ==' })).toThrow(); + expect(() => receiver.finish({ snapshotBytes: 2 })).toThrow('Incomplete'); +}); diff --git a/packages/api/src/agents/remoteSdk/snapshot.ts b/packages/api/src/agents/remoteSdk/snapshot.ts new file mode 100644 index 0000000000..4896245186 --- /dev/null +++ b/packages/api/src/agents/remoteSdk/snapshot.ts @@ -0,0 +1,76 @@ +import { z } from 'zod'; +import type { RemoteSdkSnapshot, RemoteSdkLimits } from './types'; +import { encodeSdkValue, decodeSdkValue } from './wire'; + +/** One bounded transfer per execution; chunks precede the terminal receipt in the ordered durable stream. */ +export function createSnapshotReceiver(maxBytes: number) { + let buffer: Buffer | undefined; + let received = 0; + let total: number | undefined; + let finished = false; + return { + accept(data: unknown): null { + const chunk = z + .object({ + offset: z.number().int().nonnegative(), + total: z.number().int().positive().max(maxBytes), + data: z + .string() + .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/), + }) + .parse(data); + const bytes = Buffer.from(chunk.data, 'base64'); + if ( + finished || + !bytes.length || + chunk.offset !== received || + (total !== undefined && total !== chunk.total) || + received + bytes.length > chunk.total + ) + throw new Error('Invalid SDK snapshot transfer'); + total = chunk.total; + buffer ??= Buffer.alloc(chunk.total); + bytes.copy(buffer, received); + received += bytes.length; + return null; + }, + finish(data: unknown): RemoteSdkSnapshot { + const receipt = z + .object({ snapshotBytes: z.number().int().positive().max(maxBytes) }) + .parse(data); + if (finished || received !== total || receipt.snapshotBytes !== total) + throw new Error('Incomplete SDK snapshot transfer'); + finished = true; + const snapshot = decodeSdkValue(JSON.parse(buffer!.toString('utf8'))) as RemoteSdkSnapshot; + buffer = undefined; + return snapshot; + }, + }; +} + +export async function sendSnapshot( + peer: { send: (frame: { type: 'snapshot' | 'done'; data: unknown }) => void }, + snapshot: RemoteSdkSnapshot, + limits: RemoteSdkLimits, + flush?: () => Promise, +): Promise { + const bytes = Buffer.from(JSON.stringify(encodeSdkValue(snapshot))); + if (bytes.length > limits.maxBufferedBytes) throw new Error('SDK snapshot capacity exceeded'); + // Leave room for base64 expansion, RPC identity and the wire value tags. + const chunkBytes = Math.max(1, Math.floor(limits.maxFrameBytes / 4)); + for (let offset = 0; offset < bytes.length; offset += chunkBytes) { + peer.send({ + type: 'snapshot', + data: { + offset, + total: bytes.length, + data: bytes.subarray(offset, offset + chunkBytes).toString('base64'), + }, + }); + await flush?.(); + } + peer.send({ + type: 'done', + data: { snapshotBytes: bytes.length, interrupt: snapshot.interrupt == null ? undefined : true }, + }); +} diff --git a/packages/api/src/agents/remoteSdk/types.ts b/packages/api/src/agents/remoteSdk/types.ts index 4a73adb8fc..b7a81fd0da 100644 --- a/packages/api/src/agents/remoteSdk/types.ts +++ b/packages/api/src/agents/remoteSdk/types.ts @@ -10,6 +10,8 @@ import type { import type { SdkAgentIdentity } from './identity'; export interface RemoteSdkRecoveryOptions { + /** Operator-allowlisted direct worker routes, keyed by stable owner identity. */ + routes?: Record; maxDisconnectMs: number; retryDelayMs: number; maxJournalBytes: number; @@ -50,6 +52,7 @@ export type SdkRunInput = Parameters[0]; export type SdkRunOptions = Parameters[1]; export type SdkEventData = Parameters[1]; export interface RemoteSdkStart { + transportVersion: 2; hostTools?: HostToolDescriptor[]; operation: 'start' | 'resume'; input: IState; diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 3a145b27ea..795aad4e43 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -3554,7 +3554,14 @@ describe('remote SDK runtime configuration', () => { retryDelayMs: 250, maxJournalBytes: 67108864, }); + const routed = agentsEndpointSchema.parse( + remote({ routes: { 'worker-a': 'wss://worker-a.internal/sdk' } }), + )?.sdkRuntimes?.test; + expect(routed && 'remote' in routed ? routed.remote.recovery?.routes : undefined).toEqual({ + 'worker-a': 'wss://worker-a.internal/sdk', + }); for (const recovery of [ + { routes: { 'worker-a': 'https://worker-a.internal/sdk' } }, { maxDisconnectMs: 0 }, { maxDisconnectMs: 60000 }, { retryDelayMs: -1 }, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 449f1a171d..7dc340ead1 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1913,6 +1913,15 @@ export const agentsEndpointSchema = baseEndpointSchema secret: z.string().min(1), recovery: z .object({ + routes: z + .record( + z.string().min(1).max(128), + z + .string() + .url() + .regex(/^wss?:\/\//), + ) + .optional(), maxDisconnectMs: z.number().int().min(100).max(120000).default(15000), retryDelayMs: z.number().int().min(10).max(10000).default(250), maxJournalBytes: z