diff --git a/CONTEXT.md b/CONTEXT.md index 3d27d23cae..730c48f272 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -5,4 +5,5 @@ - **Subagent thread**: a durable, view-only child conversation owned by one parent conversation and subagent identity. A parent agent may continue it by stable `threadId`; each continuation uses a fresh execution lease restored from the canonical child transcript. It is not an ordinary human-writable chat. - **Live subagent task owner**: the one API process holding a detached child execution, its abort controller, and its bounded control queue. Redis may route trusted poll/control envelopes to that owner, but it does not migrate or persist the executor; Mongo persists only the logical child thread and its continuation fence. - **Subagent completion wakeup**: a durable internal `continue` trigger pre-registered before detached child execution so a process crash cannot lose the wakeup. Delivery defers until the child's terminal transcript is persisted, targets the initiating agent and exact parent response branch, carries task metadata rather than child output, waits for the parent generation to settle, and starts the parent turn that collects the result through the existing task store. +- **Subagent activity stream**: an observational, task-scoped live projection of bounded child progress for the currently open private panel. It may cross API replicas through Redis, never carries hidden reasoning text, and never controls or settles execution. The durable child thread remains canonical and its existing polling view is the fallback for missed or unavailable live events. - **Theme definition**: a versioned, data-only description of LibreChat semantic colors and shared appearance roles, optionally specialized by light or dark mode. The theme module validates and resolves partial definitions against bundled defaults before adapters apply them. A theme definition does not contain arbitrary CSS, application behavior, or alternate feature layouts. diff --git a/api/server/routes/__test-utils__/convos-route-mocks.js b/api/server/routes/__test-utils__/convos-route-mocks.js index b739ffb893..e40b577eab 100644 --- a/api/server/routes/__test-utils__/convos-route-mocks.js +++ b/api/server/routes/__test-utils__/convos-route-mocks.js @@ -1,7 +1,9 @@ const archiveAllHandler = jest.fn(); +const subagentActivityHandlerInputs = []; module.exports = { archiveAllHandler, + subagentActivityHandlerInputs, agents: () => ({ sleep: jest.fn() }), @@ -33,6 +35,10 @@ module.exports = { return archiveAllHandler; }), createSubagentThreadViewHandler: jest.fn(() => (_req, res) => res.status(200).json({})), + createSubagentActivityStreamHandler: jest.fn((deps, stream) => { + subagentActivityHandlerInputs.push({ deps, stream }); + return (_req, res) => res.status(200).end(); + }), deleteConvoSharedLinksWithCleanup: jest.fn(), deleteAllSharedLinksWithCleanup: jest.fn(), deleteAgentCheckpoints: jest.fn(), @@ -124,6 +130,7 @@ module.exports = { assistantEndpoint: () => ({ initializeClient: jest.fn() }), subagentThreadStore: () => ({ + subscribeActivity: jest.fn(), cancelAndDrainForOwner: jest.fn().mockResolvedValue(undefined), withOwnerDeletionFence: jest.fn().mockImplementation(async (_userId, _tenantId, deletion) => { return deletion(); diff --git a/api/server/routes/__tests__/convos.spec.js b/api/server/routes/__tests__/convos.spec.js index b3ff861f97..d4948ec893 100644 --- a/api/server/routes/__tests__/convos.spec.js +++ b/api/server/routes/__tests__/convos.spec.js @@ -2,7 +2,7 @@ const express = require('express'); const request = require('supertest'); const MOCKS = '../__test-utils__/convos-route-mocks'; -const { archiveAllHandler } = require(MOCKS); +const { archiveAllHandler, subagentActivityHandlerInputs } = require(MOCKS); jest.mock('@librechat/agents', () => require(MOCKS).agents()); jest.mock('@librechat/api', () => @@ -75,6 +75,19 @@ describe('Convos Routes', () => { jest.clearAllMocks(); }); + it('binds the activity subscription adapter to the subagent task store', () => { + const binding = subagentActivityHandlerInputs.at(-1); + const subscriber = { onEvent: jest.fn() }; + + binding.stream.subscribe('child-thread', 'task-1', subscriber); + + expect(subagentThreadStore.subscribeActivity).toHaveBeenCalledWith( + 'child-thread', + 'task-1', + subscriber, + ); + }); + describe('GET /:conversationId', () => { it('returns an ordinary owned conversation', async () => { getConvo.mockResolvedValue({ conversationId: 'ordinary', title: 'Ordinary' }); diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index d343d82521..86e3981bbc 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -5,6 +5,7 @@ const { isEnabled, deleteAgentCheckpoints, createArchiveAllHandler, + createSubagentActivityStreamHandler, createSubagentThreadViewHandler, resolveImportMaxFileSize, restoreTenantContextFromReq, @@ -48,6 +49,16 @@ const filterConversationTitle = createContentFilter({ getFilters: (req) => req.config?.filters, extract: (req) => extractConversationTitleContent(req.body), }); +const subagentActivityStreamHandler = createSubagentActivityStreamHandler( + { + getConvoOwnership: db.getConvoOwnership, + getSubagentThreadForParent: db.getSubagentThreadForParent, + getMessages: db.getMessages, + }, + { + subscribe: subagentThreadTaskStore.subscribeActivity.bind(subagentThreadTaskStore), + }, +); router.use(requireJwtAuth); const isValidProjectFilter = (projectId) => @@ -94,6 +105,10 @@ router.get('/', async (req, res) => { } }); +router.get( + '/:parentConversationId/subagents/:threadId/tasks/:taskId/activity', + subagentActivityStreamHandler, +); router.get('/:parentConversationId/subagents/:threadId', subagentThreadViewHandler); router.get('/:conversationId', async (req, res) => { diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.js b/api/server/services/Endpoints/agents/subagentThreadStore.js index d071990245..534629a38a 100644 --- a/api/server/services/Endpoints/agents/subagentThreadStore.js +++ b/api/server/services/Endpoints/agents/subagentThreadStore.js @@ -7,6 +7,8 @@ const { createSubagentThreadTaskStore, createSubagentCompletionWakeupHandler, RedisSubagentTaskControlTransport, + RedisEventTransport, + SubagentActivityStream, } = require('@librechat/api'); const db = require('~/models'); const { enqueueAgentTrigger } = require('../../Agents/triggers'); @@ -46,6 +48,12 @@ const subagentThreadTaskStore = createSubagentThreadTaskStore( }, ); +registerShutdownTask( + 'subagent activity streams prepare', + () => subagentThreadTaskStore.prepareActivityForShutdown(), + { phase: 'pre-drain', priority: 100 }, +); + let taskRoutingConfigured = false; /** Starts the optional Redis owner directory before HTTP admission opens. */ @@ -62,14 +70,21 @@ async function configureSubagentTaskRouting() { * steer the caller was told had failed could still reach the child. Failing fast * turns that into the honest `unavailable` the caller already handles. */ const publisher = duplicateIoRedisClient(ioredisClient, { enableOfflineQueue: false }); + const activitySubscriber = ioredisClient.duplicate(); + const activityPublisher = duplicateIoRedisClient(ioredisClient, { enableOfflineQueue: false }); const transport = new RedisSubagentTaskControlTransport(publisher, subscriber, { namespace: cacheConfig.REDIS_KEY_PREFIX, }); try { await subagentThreadTaskStore.configureTaskControlTransport(transport); + subagentThreadTaskStore.configureActivityStream( + new SubagentActivityStream(new RedisEventTransport(activityPublisher, activitySubscriber)), + ); } catch (error) { subscriber.disconnect(); publisher.disconnect(); + activitySubscriber.disconnect(); + activityPublisher.disconnect(); throw error; } taskRoutingConfigured = true; @@ -77,7 +92,10 @@ async function configureSubagentTaskRouting() { 'subagent task control transport', async () => { await subagentThreadTaskStore.destroyTaskControlTransport(); + subagentThreadTaskStore.destroyActivityStream(); publisher.disconnect(); + activitySubscriber.disconnect(); + activityPublisher.disconnect(); }, { priority: 90 }, ); diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js new file mode 100644 index 0000000000..a75ae5c2a8 --- /dev/null +++ b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js @@ -0,0 +1,93 @@ +const mockTaskStore = { + configureTaskControlTransport: jest.fn().mockResolvedValue(undefined), + configureActivityStream: jest.fn(), + prepareActivityForShutdown: jest.fn(), + destroyTaskControlTransport: jest.fn().mockResolvedValue(undefined), + destroyActivityStream: jest.fn(), +}; + +jest.mock('@librechat/api', () => ({ + cacheConfig: { USE_REDIS: true, REDIS_KEY_PREFIX: 'test:' }, + ioredisClient: { duplicate: jest.fn() }, + isEnabled: jest.fn(() => false), + registerShutdownTask: jest.fn(), + duplicateIoRedisClient: jest.fn(), + createSubagentThreadTaskStore: jest.fn(() => mockTaskStore), + createSubagentCompletionWakeupHandler: jest.fn(), + RedisSubagentTaskControlTransport: jest.fn(), + RedisEventTransport: jest.fn(), + SubagentActivityStream: jest.fn(), +})); + +jest.mock('~/models', () => ({ + acquireSubagentThreadLease: jest.fn(), + claimSubagentTaskResult: jest.fn(), + releaseSubagentTaskResultClaim: jest.fn(), + countActiveSubagentThreadLeases: jest.fn(), + deleteConvos: jest.fn(), + deleteMessages: jest.fn(), + getConvo: jest.fn(), + getMessages: jest.fn(), + listActiveSubagentThreadLeases: jest.fn(), + releaseSubagentThreadLease: jest.fn(), + reserveSubagentThread: jest.fn(), + renewSubagentThreadLease: jest.fn(), + saveConvo: jest.fn(), + saveMessage: jest.fn(), + isSubagentOwnerAdmissible: jest.fn(), + fenceSubagentAdmission: jest.fn(), + renewSubagentAdmission: jest.fn(), + releaseSubagentAdmission: jest.fn(), +})); + +jest.mock('../../Agents/triggers', () => ({ + enqueueAgentTrigger: jest.fn(), +})); + +const { ioredisClient, registerShutdownTask, duplicateIoRedisClient } = require('@librechat/api'); +const { configureSubagentTaskRouting } = require('./subagentThreadStore'); +const activityPrepareRegistration = registerShutdownTask.mock.calls.find( + ([name]) => name === 'subagent activity streams prepare', +); + +describe('subagent thread Redis lifecycle', () => { + it('closes activity SSE before drain and disconnects its subscriber after drain', async () => { + const taskSubscriber = { disconnect: jest.fn() }; + const activitySubscriber = { disconnect: jest.fn() }; + const taskPublisher = { disconnect: jest.fn() }; + const activityPublisher = { disconnect: jest.fn() }; + ioredisClient.duplicate + .mockReturnValueOnce(taskSubscriber) + .mockReturnValueOnce(activitySubscriber); + duplicateIoRedisClient + .mockReturnValueOnce(taskPublisher) + .mockReturnValueOnce(activityPublisher); + + await configureSubagentTaskRouting(); + + expect(activityPrepareRegistration).toEqual([ + 'subagent activity streams prepare', + expect.any(Function), + { phase: 'pre-drain', priority: 100 }, + ]); + expect(registerShutdownTask).toHaveBeenCalledWith( + 'subagent task control transport', + expect.any(Function), + { priority: 90 }, + ); + const prepare = activityPrepareRegistration[1]; + prepare(); + expect(mockTaskStore.prepareActivityForShutdown).toHaveBeenCalledTimes(1); + + const shutdown = registerShutdownTask.mock.calls.find( + ([name]) => name === 'subagent task control transport', + )[1]; + await shutdown(); + + expect(mockTaskStore.destroyTaskControlTransport).toHaveBeenCalledTimes(1); + expect(mockTaskStore.destroyActivityStream).toHaveBeenCalledTimes(1); + expect(taskPublisher.disconnect).toHaveBeenCalledTimes(1); + expect(activitySubscriber.disconnect).toHaveBeenCalledTimes(1); + expect(activityPublisher.disconnect).toHaveBeenCalledTimes(1); + }); +}); diff --git a/client/src/components/Chat/Subagents/SubagentThreadPanel.test.tsx b/client/src/components/Chat/Subagents/SubagentThreadPanel.test.tsx index 8cf79862af..46dc8808aa 100644 --- a/client/src/components/Chat/Subagents/SubagentThreadPanel.test.tsx +++ b/client/src/components/Chat/Subagents/SubagentThreadPanel.test.tsx @@ -13,12 +13,23 @@ import { initSubagentAggregatorState, initSubagentTickerState } from '~/utils/su import SubagentThreadPanel from './SubagentThreadPanel'; const mockUseSubagentThreadQuery = jest.fn(); +const mockUseSubagentActivityStream = jest.fn(); const mockApprovalProviderMounted = jest.fn(); const mockApprovalProviderUnmounted = jest.fn(); let mockIsMobile = false; jest.mock('~/data-provider', () => ({ useSubagentThreadQuery: (...args: unknown[]) => mockUseSubagentThreadQuery(...args), + subagentThreadHasTaskEvidence: (view: SubagentThreadView | undefined, taskId: string): boolean => + view?.messages.some( + (message) => + message.messageId === `${taskId}:user` || message.messageId === `${taskId}:assistant`, + ) === true, +})); + +jest.mock('~/data-provider/Subagents/useSubagentActivityStream', () => ({ + __esModule: true, + default: (...args: unknown[]) => mockUseSubagentActivityStream(...args), })); jest.mock('~/hooks', () => ({ @@ -156,6 +167,7 @@ describe('SubagentThreadPanel', () => { 'child-thread', 'task', ); + expect(mockUseSubagentActivityStream).toHaveBeenCalledWith(selection, false); expect(screen.getByText('Research child')).toBeInTheDocument(); expect(screen.getByText('Investigate the release.')).toBeInTheDocument(); expect(screen.getByText('The release is ready.')).toBeInTheDocument(); @@ -205,6 +217,46 @@ describe('SubagentThreadPanel', () => { expect(screen.getByTestId('shared-activity')).toHaveAttribute('data-state', 'ready'); }); + it('renders newer detached progress instead of a dispatch-time parent snapshot', () => { + mockUseSubagentThreadQuery.mockReturnValue({ + data: { ...completedView, status: 'running', activity: [] }, + isLoading: false, + isError: false, + isReadinessPending: false, + }); + const progressKey = subagentProgressKey( + selection.parentMessageId, + selection.toolCallId, + selection.partIndex, + ); + const detachedSelection: ActiveSubagentPanel = { + ...selection, + persistedContent: [ + { type: ContentTypes.TEXT, text: 'Dispatch-time snapshot.' }, + ] as TMessageContentParts[], + }; + + render( + + set(subagentProgressByToolCallId(progressKey), { + subagentRunId: 'child-run', + subagentType: 'researcher', + status: 'message_delta', + contentParts: [{ type: ContentTypes.TEXT, text: 'latest detached text.' }], + aggregatorState: initSubagentAggregatorState(), + tickerState: initSubagentTickerState(), + coverage: 'suffix', + }) + } + > + + , + ); + + expect(screen.getByText('Dispatch-time snapshot.latest detached text.')).toBeInTheDocument(); + }); + it('resets invocation-scoped approval state when the selected card changes', () => { mockUseSubagentThreadQuery.mockReturnValue({ data: undefined, @@ -249,6 +301,57 @@ describe('SubagentThreadPanel', () => { expect(screen.getByTestId('shared-activity')).toHaveAttribute('data-state', 'loading'); expect(screen.getByTestId('shared-activity')).toHaveAttribute('data-status', 'dispatched'); + expect(mockUseSubagentActivityStream).toHaveBeenLastCalledWith(selection, true); + }); + + it('opens live activity before the durable child becomes addressable', () => { + mockUseSubagentThreadQuery.mockReturnValue({ + data: undefined, + isLoading: false, + isError: true, + isReadinessPending: true, + }); + + const { rerender } = render( + + + , + ); + expect(mockUseSubagentActivityStream).toHaveBeenLastCalledWith(selection, true); + + mockUseSubagentThreadQuery.mockReturnValue({ + data: { ...completedView, status: 'running' }, + isLoading: false, + isError: false, + isReadinessPending: false, + }); + rerender( + + + , + ); + + expect(mockUseSubagentActivityStream).toHaveBeenLastCalledWith(selection, true); + }); + + it('keeps streaming when terminal thread state belongs to an older task', () => { + mockUseSubagentThreadQuery.mockReturnValue({ + data: { + ...completedView, + messages: [{ ...completedView.messages[1], messageId: 'older-task:assistant' }], + }, + isLoading: false, + isError: false, + isReadinessPending: false, + }); + + render( + + + , + ); + + expect(mockUseSubagentActivityStream).toHaveBeenLastCalledWith(selection, true); }); it('surfaces a durable read failure after the readiness window', () => { diff --git a/client/src/components/Chat/Subagents/SubagentThreadPanel.tsx b/client/src/components/Chat/Subagents/SubagentThreadPanel.tsx index b2ff2151ad..4ef8aac190 100644 --- a/client/src/components/Chat/Subagents/SubagentThreadPanel.tsx +++ b/client/src/components/Chat/Subagents/SubagentThreadPanel.tsx @@ -8,9 +8,10 @@ import { subagentProgressByToolCallId, subagentProgressKey, } from '~/store/subagents'; +import useSubagentActivityStream from '~/data-provider/Subagents/useSubagentActivityStream'; +import { subagentThreadHasTaskEvidence, useSubagentThreadQuery } from '~/data-provider'; import { adaptDurableThreadActivity, adaptLivePersistedActivity } from './adapters'; import ApprovalProvider from '~/components/Chat/Messages/Content/ApprovalContext'; -import { useSubagentThreadQuery } from '~/data-provider'; import { useFocusTrap, useLocalize } from '~/hooks'; import SubagentActivity from './SubagentActivity'; @@ -35,6 +36,13 @@ export default function SubagentThreadPanel({ selection }: { selection: ActiveSu threadId, taskId, ); + const durableTerminal = + subagentThreadHasTaskEvidence(data, taskId) && + (data?.status === 'completed' || + data?.status === 'failed' || + data?.status === 'interrupted' || + data?.status === 'cancelled'); + useSubagentActivityStream(selection, !durableTerminal); const detachedLiveSubmitting = selection.durable != null && progress != null && @@ -73,6 +81,7 @@ export default function SubagentThreadPanel({ selection }: { selection: ActiveSu prompt: selection.prompt, progress, persistedContent: selection.persistedContent, + isDetached: selection.durable != null, legacyOutput: selection.legacyOutput, // A detached parent tool step closes as soon as dispatch succeeds; // its terminal status does not describe the still-running child. diff --git a/client/src/components/Chat/Subagents/adapters.test.ts b/client/src/components/Chat/Subagents/adapters.test.ts index 273efb613a..bb983d9fb9 100644 --- a/client/src/components/Chat/Subagents/adapters.test.ts +++ b/client/src/components/Chat/Subagents/adapters.test.ts @@ -45,6 +45,123 @@ describe('child activity adapters', () => { ]); }); + it('merges a forward-only detached suffix with the partial parent snapshot', () => { + const activity = adaptLivePersistedActivity({ + title: 'researcher', + progress: { + subagentRunId: 'run', + subagentType: 'researcher', + status: 'message_delta', + contentParts: [{ type: ContentTypes.TEXT, text: 'latest detached text.' }], + aggregatorState: initSubagentAggregatorState(), + tickerState: initSubagentTickerState(), + coverage: 'suffix', + }, + persistedContent: [ + { type: ContentTypes.TEXT, text: 'Dispatch-time snapshot; ' }, + ] as TMessageContentParts[], + initialProgress: 1, + isSubmitting: true, + isDetached: true, + }); + + expect(activity.items).toEqual([ + { type: 'writing', text: 'Dispatch-time snapshot; latest detached text.' }, + ]); + }); + + it('uses a complete parent-stream projection without duplicating persistence', () => { + const activity = adaptLivePersistedActivity({ + title: 'researcher', + progress: { + subagentRunId: 'run', + subagentType: 'researcher', + status: 'message_delta', + contentParts: [{ type: ContentTypes.TEXT, text: 'Complete live text.' }], + aggregatorState: initSubagentAggregatorState(), + tickerState: initSubagentTickerState(), + coverage: 'complete', + }, + persistedContent: [{ type: ContentTypes.TEXT, text: 'Complete ' }] as TMessageContentParts[], + initialProgress: 1, + isSubmitting: true, + isDetached: true, + }); + + expect(activity.items).toEqual([{ type: 'writing', text: 'Complete live text.' }]); + }); + + it('appends coincident text in a forward-only suffix', () => { + const activity = adaptLivePersistedActivity({ + title: 'researcher', + progress: { + subagentRunId: 'run', + subagentType: 'researcher', + status: 'message_delta', + contentParts: [{ type: ContentTypes.TEXT, text: 'ha' }], + aggregatorState: initSubagentAggregatorState(), + tickerState: initSubagentTickerState(), + coverage: 'suffix', + }, + persistedContent: [{ type: ContentTypes.TEXT, text: 'ha' }] as TMessageContentParts[], + initialProgress: 1, + isSubmitting: true, + isDetached: true, + }); + + expect(activity.items).toEqual([{ type: 'writing', text: 'haha' }]); + }); + + it('preserves persisted tool fields when a sparse completion is the live suffix', () => { + const activity = adaptLivePersistedActivity({ + title: 'researcher', + progress: { + subagentRunId: 'run', + subagentType: 'researcher', + status: 'run_step_completed', + contentParts: [ + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: 'tool-1', + name: '', + args: '{}', + output: 'Found it.', + progress: 1, + }, + }, + ], + aggregatorState: initSubagentAggregatorState(), + tickerState: initSubagentTickerState(), + coverage: 'suffix', + }, + persistedContent: [ + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: 'tool-1', + name: 'search', + args: '{"query":"release"}', + progress: 0.1, + }, + }, + ] as unknown as TMessageContentParts[], + initialProgress: 1, + isSubmitting: true, + isDetached: true, + }); + + expect(activity.items).toEqual([ + expect.objectContaining({ + type: 'tool', + name: 'search', + input: '{"query":"release"}', + output: 'Found it.', + status: 'completed', + }), + ]); + }); + it('rehydrates the selected detached task from its sanitized durable activity', () => { const view: SubagentThreadView = { threadId: 'thread', diff --git a/client/src/components/Chat/Subagents/adapters.ts b/client/src/components/Chat/Subagents/adapters.ts index 55452a82f4..b6abb4a525 100644 --- a/client/src/components/Chat/Subagents/adapters.ts +++ b/client/src/components/Chat/Subagents/adapters.ts @@ -102,6 +102,50 @@ const publicActivityToChildActivity = (items: SubagentActivityItem[]): ChildActi }; }); +/** Merge activity whose transport explicitly declares it is a forward-only + * suffix. Complete parent-stream projections bypass this function entirely. */ +const mergePersistedAndLiveActivity = ( + persisted: ChildActivityItem[], + live: ChildActivityItem[], +): ChildActivityItem[] => { + if (persisted.length === 0) return live; + if (live.length === 0) return persisted; + + const merged = [...persisted]; + for (const item of live) { + if (item.type === 'tool') { + const existingIndex = merged.findIndex( + (candidate) => candidate.type === 'tool' && candidate.toolCallId === item.toolCallId, + ); + if (existingIndex >= 0) { + const existing = merged[existingIndex] as Extract; + const next = { ...existing, ...item }; + if (item.name === '') next.name = existing.name; + if ( + existing.input != null && + (item.input == null || item.input === '' || item.input === '{}') + ) { + next.input = existing.input; + } + merged[existingIndex] = next; + } else { + merged.push(item); + } + continue; + } + + const previous = merged.at(-1); + if (previous?.type !== item.type) { + merged.push(item); + continue; + } + const previousText = previous.text ?? ''; + const nextText = item.text ?? ''; + merged[merged.length - 1] = { ...item, text: `${previousText}${nextText}` }; + } + return merged; +}; + const liveStatus = ({ progress, initialProgress, @@ -131,17 +175,22 @@ export function adaptLivePersistedActivity(input: { initialProgress: number; isSubmitting: boolean; runStepStatus?: PartMetadata['runStepStatus']; + isDetached?: boolean; reasoningVisibility?: 'visible' | 'marker'; approvalVisibility?: 'visible' | 'hidden'; }): ChildActivity { const persisted = input.persistedContent ?? []; const live = (input.progress?.contentParts ?? []) as TMessageContentParts[]; - const parts = persisted.length > 0 ? persisted : live; - const items = contentPartsToActivity( - parts, - input.reasoningVisibility ?? 'visible', - input.approvalVisibility ?? 'visible', - ); + const reasoningVisibility = input.reasoningVisibility ?? 'visible'; + const approvalVisibility = input.approvalVisibility ?? 'visible'; + const persistedItems = contentPartsToActivity(persisted, reasoningVisibility, approvalVisibility); + const liveItems = contentPartsToActivity(live, reasoningVisibility, approvalVisibility); + let items = persistedItems.length > 0 ? persistedItems : liveItems; + if (input.isDetached === true && input.progress?.coverage === 'suffix') { + items = mergePersistedAndLiveActivity(persistedItems, liveItems); + } else if (input.isDetached === true && liveItems.length > 0) { + items = liveItems; + } if (items.length === 0 && input.legacyOutput != null && input.legacyOutput !== '') { items.push({ type: 'writing', text: input.legacyOutput }); } diff --git a/client/src/data-provider/Subagents/queries.test.ts b/client/src/data-provider/Subagents/queries.test.ts index 61bdd883ec..0c464f04be 100644 --- a/client/src/data-provider/Subagents/queries.test.ts +++ b/client/src/data-provider/Subagents/queries.test.ts @@ -2,6 +2,7 @@ import { renderHook } from '@testing-library/react'; import type { SubagentThreadView } from 'librechat-data-provider'; import { isSubagentReadinessPending, + subagentThreadHasTaskEvidence, subagentThreadRefetchInterval, useSubagentThreadQuery, } from './queries'; @@ -46,6 +47,16 @@ describe('subagent thread refresh policy', () => { expect(subagentThreadRefetchInterval(prior, 1_000, 1_000, 'new-task')).toBe(false); }); + it('associates terminal state only with evidence from the selected task', () => { + const prior = { + ...view('completed'), + messages: [{ messageId: 'old-task:assistant' }], + } as SubagentThreadView; + + expect(subagentThreadHasTaskEvidence(prior, 'new-task')).toBe(false); + expect(subagentThreadHasTaskEvidence(prior, 'old-task')).toBe(true); + }); + it('stops polling an older API view once the exact task response exists', () => { const rollingDeployView = { ...view('running'), diff --git a/client/src/data-provider/Subagents/queries.ts b/client/src/data-provider/Subagents/queries.ts index 81b102e3cc..bb626cb345 100644 --- a/client/src/data-provider/Subagents/queries.ts +++ b/client/src/data-provider/Subagents/queries.ts @@ -13,20 +13,22 @@ const isTerminal = (status: SubagentThreadView['status']): boolean => status === 'interrupted' || status === 'cancelled'; +export const subagentThreadHasTaskEvidence = ( + view: SubagentThreadView | undefined, + taskId: string, +): boolean => + view?.messages.some( + (message) => + message.messageId === `${taskId}:user` || message.messageId === `${taskId}:assistant`, + ) === true; + export const subagentThreadRefetchInterval = ( view: SubagentThreadView | undefined, readinessDeadline: number, now = Date.now(), expectedTaskId?: string, ): number | false => { - if ( - expectedTaskId != null && - !view?.messages.some( - (message) => - message.messageId === `${expectedTaskId}:user` || - message.messageId === `${expectedTaskId}:assistant`, - ) - ) { + if (expectedTaskId != null && !subagentThreadHasTaskEvidence(view, expectedTaskId)) { return now < readinessDeadline ? ACTIVE_THREAD_REFRESH_MS : false; } // During a rolling deploy, an older replica can return a thread-wide status diff --git a/client/src/data-provider/Subagents/useSubagentActivityStream.test.tsx b/client/src/data-provider/Subagents/useSubagentActivityStream.test.tsx new file mode 100644 index 0000000000..f09f96d18a --- /dev/null +++ b/client/src/data-provider/Subagents/useSubagentActivityStream.test.tsx @@ -0,0 +1,303 @@ +import React from 'react'; +import { act, renderHook } from '@testing-library/react'; +import { RecoilRoot, useRecoilValue, useSetRecoilState } from 'recoil'; +import { ContentTypes, QueryKeys, StepEvents } from 'librechat-data-provider'; +import type { ActiveSubagentPanel } from '~/store/subagents'; +import { + subagentParentStreamOpenByToolCallId, + subagentProgressByToolCallId, + subagentProgressKey, + takeRegisteredSubagentProgressKeys, +} from '~/store/subagents'; +import useSubagentActivityStream from './useSubagentActivityStream'; + +type Listener = (event: MessageEvent) => void; +type MockStream = { + url: string; + options: { method?: string; headers?: Record }; + listeners: Record; + close: jest.Mock; + emit: (type: string, data: unknown) => void; +}; + +const streams: MockStream[] = []; +jest.mock('sse.js', () => ({ + SSE: jest.fn().mockImplementation((url: string, options: MockStream['options']) => { + const listeners: Record = {}; + const stream: MockStream = { + url, + options, + listeners, + close: jest.fn(), + emit: (type, data) => listeners[type]?.({ data: JSON.stringify(data) } as MessageEvent), + }; + streams.push(stream); + return { + addEventListener: (type: string, listener: Listener) => { + listeners[type] = listener; + }, + close: stream.close, + }; + }), +})); + +const mockInvalidateQueries = jest.fn(); +const mockQueryClient = { invalidateQueries: mockInvalidateQueries }; +jest.mock('@tanstack/react-query', () => ({ + ...jest.requireActual('@tanstack/react-query'), + useQueryClient: () => mockQueryClient, +})); + +jest.mock('~/hooks/AuthContext', () => ({ + useAuthContext: () => ({ token: 'token-1', isAuthenticated: true }), +})); + +const selection: ActiveSubagentPanel = { + host: 'conversation', + parentConversationId: 'parent conversation', + parentMessageId: 'parent-message', + toolCallId: 'tool-call', + partIndex: 1, + subagentType: 'researcher', + initialProgress: 1, + isSubmitting: false, + durable: { threadId: 'child/thread', taskId: 'task?1' }, +}; + +const wrapper = ({ children }: { children: React.ReactNode }) => ( + {children} +); + +describe('useSubagentActivityStream', () => { + beforeEach(() => { + streams.length = 0; + mockInvalidateQueries.mockClear(); + takeRegisteredSubagentProgressKeys(); + }); + + it('opens one authorized task stream and closes after terminal delivery', () => { + const { result, unmount } = renderHook( + () => { + useSubagentActivityStream(selection); + return useRecoilValue( + subagentProgressByToolCallId( + subagentProgressKey( + selection.parentMessageId, + selection.toolCallId, + selection.partIndex, + ), + ), + ); + }, + { wrapper }, + ); + + expect(streams).toHaveLength(1); + expect(streams[0]?.url).toContain( + '/api/convos/parent%20conversation/subagents/child%2Fthread/tasks/task%3F1/activity', + ); + expect(streams[0]?.options.headers).toEqual({ Authorization: 'Bearer token-1' }); + + act(() => { + streams[0]?.emit('message', { + event: StepEvents.ON_SUBAGENT_UPDATE, + data: { + runId: 'root', + parentRunId: 'parent', + subagentRunId: 'child', + activityEventId: 'task-1:0', + activitySequence: 0, + subagentType: 'researcher', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: ['parent'], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'Live child output' }] } }, + timestamp: '2026-08-21T20:00:00.000Z', + }, + }); + streams[0]?.emit('message', { + final: true, + subagentActivity: true, + status: 'completed', + }); + }); + + expect(result.current?.contentParts).toEqual([{ type: 'text', text: 'Live child output' }]); + expect(result.current?.coverage).toBe('complete'); + expect(takeRegisteredSubagentProgressKeys()).toEqual([ + subagentProgressKey(selection.parentMessageId, selection.toolCallId, selection.partIndex), + ]); + expect(streams[0]?.close).toHaveBeenCalledTimes(1); + expect(mockInvalidateQueries).toHaveBeenCalledWith([ + QueryKeys.subagentThread, + 'parent conversation', + 'child/thread', + 'task?1', + ]); + unmount(); + expect(streams[0]?.close).toHaveBeenCalledTimes(1); + }); + + it('accepts an exact task-stream update when older providers omit the optional tool-call id', () => { + const { result } = renderHook( + () => { + useSubagentActivityStream(selection); + return useRecoilValue( + subagentProgressByToolCallId( + subagentProgressKey( + selection.parentMessageId, + selection.toolCallId, + selection.partIndex, + ), + ), + ); + }, + { wrapper }, + ); + + act(() => { + streams[0]?.emit('message', { + event: StepEvents.ON_SUBAGENT_UPDATE, + data: { + runId: 'root', + parentRunId: 'parent', + subagentRunId: 'child', + subagentType: 'researcher', + subagentKind: 'agent', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'Compatible update' }] } }, + timestamp: '2026-08-21T20:00:00.000Z', + }, + }); + }); + + expect(result.current?.contentParts).toEqual([{ type: 'text', text: 'Compatible update' }]); + }); + + it('buffers the first detached suffix while the parent stream is still open', () => { + const activeSelection = { ...selection, isSubmitting: true }; + const key = subagentProgressKey( + activeSelection.parentMessageId, + activeSelection.toolCallId, + activeSelection.partIndex, + ); + const { result } = renderHook( + () => { + useSubagentActivityStream(activeSelection); + return { + progress: useRecoilValue(subagentProgressByToolCallId(key)), + parentOpen: useRecoilValue(subagentParentStreamOpenByToolCallId(key)), + closeParent: useSetRecoilState(subagentParentStreamOpenByToolCallId(key)), + }; + }, + { wrapper }, + ); + + act(() => { + streams[0]?.emit('message', { + event: StepEvents.ON_SUBAGENT_UPDATE, + data: { + runId: 'root', + parentRunId: 'parent', + subagentRunId: 'child', + activityEventId: 'task-1:5', + activitySequence: 5, + subagentType: 'researcher', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'suffix' }] } }, + timestamp: '2026-08-21T20:00:00.000Z', + }, + }); + }); + + expect(result.current.parentOpen).toBe(true); + expect(result.current.progress?.contentParts).toEqual([]); + expect(result.current.progress?.pendingSequencedEvents).toHaveLength(1); + + act(() => result.current.closeParent(false)); + + expect(result.current.parentOpen).toBe(false); + expect(result.current.progress?.contentParts).toEqual([ + { type: ContentTypes.TEXT, text: 'suffix' }, + ]); + expect(result.current.progress?.pendingSequencedEvents).toBeUndefined(); + }); + + it('reconnects with bounded backoff after a transient stream error', () => { + jest.useFakeTimers(); + const { unmount } = renderHook(() => useSubagentActivityStream(selection), { wrapper }); + + act(() => streams[0]?.emit('error', {})); + expect(streams[0]?.close).toHaveBeenCalledTimes(1); + expect(streams).toHaveLength(1); + + act(() => jest.advanceTimersByTime(500)); + expect(streams).toHaveLength(2); + + unmount(); + expect(streams[1]?.close).toHaveBeenCalledTimes(1); + jest.useRealTimers(); + }); + + it('preserves reconnect backoff after a stream-unavailable envelope', () => { + jest.useFakeTimers(); + const { unmount } = renderHook(() => useSubagentActivityStream(selection), { wrapper }); + + act(() => streams[0]?.emit('error', {})); + act(() => jest.advanceTimersByTime(500)); + expect(streams).toHaveLength(2); + + act(() => { + streams[1]?.emit('message', { error: 'Subagent activity stream unavailable' }); + streams[1]?.emit('error', {}); + jest.advanceTimersByTime(999); + }); + expect(streams).toHaveLength(2); + + act(() => jest.advanceTimersByTime(1)); + expect(streams).toHaveLength(3); + + unmount(); + jest.useRealTimers(); + }); + + it('keeps one forward-only stream across metadata-only selection updates', () => { + const { rerender } = renderHook(({ value }) => useSubagentActivityStream(value), { + initialProps: { value: selection }, + wrapper, + }); + expect(streams).toHaveLength(1); + + rerender({ + value: { + ...selection, + persistedContent: [{ type: ContentTypes.TEXT, text: 'New snapshot.' }], + durable: { ...selection.durable! }, + }, + }); + + expect(streams).toHaveLength(1); + expect(streams[0]?.close).not.toHaveBeenCalled(); + }); + + it('never opens the private task stream for shares or foreground children', () => { + const { rerender } = renderHook(({ value }) => useSubagentActivityStream(value), { + initialProps: { value: { ...selection, host: 'share' } as ActiveSubagentPanel }, + wrapper, + }); + expect(streams).toHaveLength(0); + + rerender({ value: { ...selection, host: 'conversation', durable: undefined } }); + expect(streams).toHaveLength(0); + }); +}); diff --git a/client/src/data-provider/Subagents/useSubagentActivityStream.ts b/client/src/data-provider/Subagents/useSubagentActivityStream.ts new file mode 100644 index 0000000000..1bce9741ee --- /dev/null +++ b/client/src/data-provider/Subagents/useSubagentActivityStream.ts @@ -0,0 +1,167 @@ +import { useEffect, useRef } from 'react'; +import { SSE } from 'sse.js'; +import { useQueryClient } from '@tanstack/react-query'; +import { useRecoilValue, useSetRecoilState } from 'recoil'; +import { QueryKeys, StepEvents, apiBaseUrl } from 'librechat-data-provider'; +import type { SubagentUpdateEvent } from 'librechat-data-provider'; +import type { ActiveSubagentPanel } from '~/store/subagents'; +import { + closeParentSubagentProgress, + reduceSubagentProgress, + registerSubagentProgressKey, + subagentParentStreamOpenByToolCallId, + subagentProgressByToolCallId, + subagentProgressKey, +} from '~/store/subagents'; +import { useAuthContext } from '~/hooks/AuthContext'; + +type ActivityEnvelope = { + event?: unknown; + data?: unknown; + final?: unknown; + subagentActivity?: unknown; +}; + +const INITIAL_RECONNECT_MS = 500; +const MAX_RECONNECT_MS = 5_000; + +const isSubagentUpdate = (value: unknown): value is SubagentUpdateEvent => { + if (value == null || typeof value !== 'object') return false; + const event = value as Partial; + return ( + typeof event.subagentRunId === 'string' && + typeof event.subagentType === 'string' && + (event.activityEventId == null || typeof event.activityEventId === 'string') && + (event.activitySequence == null || + (Number.isSafeInteger(event.activitySequence) && event.activitySequence >= 0)) && + (event.parentToolCallId == null || typeof event.parentToolCallId === 'string') && + typeof event.phase === 'string' + ); +}; + +/** Live-only enhancement for the selected durable child; the durable query remains canonical. */ +export default function useSubagentActivityStream( + selection: ActiveSubagentPanel, + enabled = true, +): void { + const { token, isAuthenticated } = useAuthContext(); + const queryClient = useQueryClient(); + const key = subagentProgressKey( + selection.parentMessageId, + selection.toolCallId, + selection.partIndex, + ); + const setProgress = useSetRecoilState(subagentProgressByToolCallId(key)); + const parentStreamOpen = useRecoilValue(subagentParentStreamOpenByToolCallId(key)); + const setParentStreamOpen = useSetRecoilState(subagentParentStreamOpenByToolCallId(key)); + const parentStreamOpenRef = useRef(parentStreamOpen); + const durable = selection.durable; + const threadId = durable?.threadId; + const taskId = durable?.taskId; + + useEffect(() => { + parentStreamOpenRef.current = parentStreamOpen; + if (!parentStreamOpen) { + setProgress(closeParentSubagentProgress); + } + }, [parentStreamOpen, setProgress]); + + useEffect(() => { + if (!selection.isSubmitting) return; + registerSubagentProgressKey(key); + parentStreamOpenRef.current = true; + setParentStreamOpen(true); + }, [key, selection.isSubmitting, setParentStreamOpen]); + + useEffect(() => { + if ( + selection.host !== 'conversation' || + threadId == null || + taskId == null || + !enabled || + !isAuthenticated || + token == null + ) { + return; + } + + const queryKey = [QueryKeys.subagentThread, selection.parentConversationId, threadId, taskId]; + const endpoint = `${apiBaseUrl()}/api/convos/${encodeURIComponent(selection.parentConversationId)}/subagents/${encodeURIComponent(threadId)}/tasks/${encodeURIComponent(taskId)}/activity`; + let stream: SSE | undefined; + let retryTimer: ReturnType | undefined; + let retryAttempt = 0; + let disposed = false; + let terminal = false; + + const closeCurrent = () => { + const current = stream; + stream = undefined; + current?.close(); + }; + const connect = () => { + retryTimer = undefined; + if (disposed || terminal) return; + const next = new SSE(endpoint, { + method: 'GET', + headers: { Authorization: `Bearer ${token}` }, + }); + stream = next; + + next.addEventListener('message', (message: MessageEvent) => { + if (stream !== next || disposed) return; + let envelope: ActivityEnvelope; + try { + envelope = JSON.parse(message.data) as ActivityEnvelope; + } catch { + return; + } + if (envelope.final === true && envelope.subagentActivity === true) { + terminal = true; + closeCurrent(); + void queryClient.invalidateQueries(queryKey); + return; + } + const event = envelope.data; + if (envelope.event !== StepEvents.ON_SUBAGENT_UPDATE || !isSubagentUpdate(event)) { + return; + } + if (event.parentToolCallId != null && event.parentToolCallId !== selection.toolCallId) { + return; + } + retryAttempt = 0; + registerSubagentProgressKey(key); + setProgress((previous) => + reduceSubagentProgress(previous, [event], 'detached', parentStreamOpenRef.current), + ); + }); + next.addEventListener('error', () => { + if (stream !== next || disposed || terminal || retryTimer != null) return; + closeCurrent(); + const delay = Math.min(INITIAL_RECONNECT_MS * 2 ** retryAttempt, MAX_RECONNECT_MS); + retryAttempt += 1; + retryTimer = setTimeout(connect, delay); + }); + }; + + connect(); + return () => { + disposed = true; + if (retryTimer != null) clearTimeout(retryTimer); + closeCurrent(); + }; + }, [ + enabled, + isAuthenticated, + key, + queryClient, + selection.host, + selection.parentConversationId, + selection.parentMessageId, + selection.partIndex, + selection.toolCallId, + setProgress, + taskId, + threadId, + token, + ]); +} diff --git a/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts b/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts index 8e6fe87ac0..81f0986dc9 100644 --- a/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts +++ b/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts @@ -3532,6 +3532,42 @@ describe('useStepHandler', () => { expect(getProgress('call_keep')).not.toBeNull(); }); + + it('uses parent stream closure to release a detached sequence waiting at handoff', () => { + const { result, getProgress } = renderStepHandlerWithReader(); + const { submission } = seedResponseWithSubagentToolCalls(result, ['call_handoff']); + + act(() => { + (result.current as any).stepHandler( + { + event: StepEvents.ON_SUBAGENT_UPDATE, + data: makeUpdate({ + parentToolCallId: 'call_handoff', + activityEventId: 'task:5', + activitySequence: 5, + phase: 'message_delta', + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'suffix' }] } }, + }), + }, + submission, + ); + }); + + expect(getProgress('call_handoff')).toEqual( + expect.objectContaining({ contentParts: [], pendingSequencedEvents: [expect.any(Object)] }), + ); + + act(() => { + (result.current as any).clearStepMaps(); + }); + + expect(getProgress('call_handoff')).toEqual( + expect.objectContaining({ + contentParts: [{ type: ContentTypes.TEXT, text: 'suffix' }], + lastActivitySequence: 5, + }), + ); + }); }); /** diff --git a/client/src/hooks/SSE/useStepHandler.ts b/client/src/hooks/SSE/useStepHandler.ts index 92728a7e93..890217eacb 100644 --- a/client/src/hooks/SSE/useStepHandler.ts +++ b/client/src/hooks/SSE/useStepHandler.ts @@ -23,14 +23,14 @@ import type { import type { SetterOrUpdater } from 'recoil'; import type { AnnounceOptions } from '~/common'; import { - foldSubagentEvent, - foldSubagentEventIntoTicker, - initSubagentAggregatorState, - initSubagentTickerState, -} from '~/utils/subagentContent'; -import { + closeParentSubagentProgress, + listRegisteredSubagentProgressKeys, + reduceSubagentProgress, + registerSubagentProgressKey, + subagentParentStreamOpenByToolCallId, subagentProgressByToolCallId, subagentProgressKey, + takeRegisteredSubagentProgressKeys, sandboxStartingByToolCallId, } from '~/store'; import { isAskUserQuestionPart, isAnsweredAskUserQuestionPart } from '~/utils/approval'; @@ -183,13 +183,6 @@ export default function useStepHandler({ const pendingSubagentBuffer = useRef( new Map(), ); - /** - * Tracked atom keys so `clearStepMaps` can reset them. Without this, each - * subagent invocation leaks an `events: SubagentUpdateEvent[]` array in the - * `atomFamily` — atoms persist for the app lifetime. - */ - const knownSubagentAtomKeys = useRef(new Set()); - const getCurrentMessages = useCallback( (messages: TMessage[]) => { const freshMessages = getMessages(); @@ -276,35 +269,11 @@ export default function useStepHandler({ } const toApply = pending ? [...pending.events, payload] : [payload]; - knownSubagentAtomKeys.current.add(invocationKey); - set(subagentProgressByToolCallId(invocationKey), (prev) => { - /** Fold the batch into both aggregators. Pure functions — they - * return a new reference only when something actually changed, - * so React bails out of unnecessary re-renders downstream. */ - let contentParts = prev?.contentParts ?? []; - let aggregatorState = prev?.aggregatorState ?? initSubagentAggregatorState(); - let tickerState = prev?.tickerState ?? initSubagentTickerState(); - for (const event of toApply) { - ({ parts: contentParts, state: aggregatorState } = foldSubagentEvent( - contentParts, - aggregatorState, - event, - )); - tickerState = foldSubagentEventIntoTicker(tickerState, event); - } - - const last = toApply[toApply.length - 1]; - return { - subagentRunId: payload.subagentRunId, - subagentType: payload.subagentType, - subagentAgentId: payload.subagentAgentId ?? prev?.subagentAgentId, - contentParts, - aggregatorState, - tickerState, - status: last.phase, - latestLabel: last.label ?? prev?.latestLabel, - }; - }); + registerSubagentProgressKey(invocationKey); + set(subagentParentStreamOpenByToolCallId(invocationKey), true); + set(subagentProgressByToolCallId(invocationKey), (prev) => + reduceSubagentProgress(prev, toApply, 'parent', true), + ); }, [resolveSubagentInvocationKey], ); @@ -323,10 +292,21 @@ export default function useStepHandler({ const resetSubagentAtoms = useRecoilCallback( ({ reset }) => (): void => { - for (const invocationKey of knownSubagentAtomKeys.current) { + for (const invocationKey of takeRegisteredSubagentProgressKeys()) { reset(subagentProgressByToolCallId(invocationKey)); + reset(subagentParentStreamOpenByToolCallId(invocationKey)); + } + }, + [], + ); + + const closeParentSubagentStreams = useRecoilCallback( + ({ set }) => + (): void => { + for (const invocationKey of listRegisteredSubagentProgressKeys()) { + set(subagentParentStreamOpenByToolCallId(invocationKey), false); + set(subagentProgressByToolCallId(invocationKey), closeParentSubagentProgress); } - knownSubagentAtomKeys.current.clear(); }, [], ); @@ -1437,6 +1417,7 @@ export default function useStepHandler({ subagentRunToInvocationKey.current.clear(); claimedSubagentInvocationKeys.current.clear(); pendingSubagentBuffer.current.clear(); + closeParentSubagentStreams(); /** Unlike subagent atoms below, sandbox-starting flags are transient * status with no audit value — reset them at this boundary so an * interrupted cold boot can't leak a stale "starting" label onto a @@ -1450,7 +1431,7 @@ export default function useStepHandler({ * persisted `subagent_content` takes over for historical messages * once the conversation is saved, and we prevent unbounded * atomFamily growth across multi-conversation sessions. */ - }, [cancelPendingDeltaFlush, resetSandboxAtoms]); + }, [cancelPendingDeltaFlush, closeParentSubagentStreams, resetSandboxAtoms]); /** * Sync a message into the step handler's messageMap. diff --git a/client/src/store/subagents.spec.ts b/client/src/store/subagents.spec.ts new file mode 100644 index 0000000000..bb122674a1 --- /dev/null +++ b/client/src/store/subagents.spec.ts @@ -0,0 +1,371 @@ +import { ContentTypes } from 'librechat-data-provider'; +import type { SubagentUpdateEvent } from 'librechat-data-provider'; +import { closeParentSubagentProgress, reduceSubagentProgress } from './subagents'; + +const update = (overrides: Partial = {}): SubagentUpdateEvent => ({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + activityEventId: 'activity-1', + subagentType: 'researcher', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'Working.' }] } }, + label: 'Drafting the report', + timestamp: '2026-08-21T20:00:00.000Z', + ...overrides, +}); + +describe('reduceSubagentProgress', () => { + it('folds an event delivered by both parent and detached streams only once', () => { + const event = update(); + const first = reduceSubagentProgress(null, [event]); + const replay = reduceSubagentProgress(first, [event]); + + expect(replay).toBe(first); + expect(first?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'Working.' }]); + expect(first?.tickerState.lines).toHaveLength(1); + }); + + it('preserves equal chunks that carry distinct host event identities', () => { + const progress = reduceSubagentProgress(null, [ + update({ activityEventId: 'activity-1', activitySequence: 0 }), + update({ activityEventId: 'activity-2', activitySequence: 1 }), + ]); + + expect(progress?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'Working.Working.' }]); + }); + + it('marks an accepted run-start frame complete regardless of its delivery transport', () => { + const progress = reduceSubagentProgress( + null, + [update({ activitySequence: 0 })], + 'detached', + false, + ); + + expect(progress?.coverage).toBe('complete'); + }); + + it('orders a same-batch overlap by the host sequence before folding', () => { + const progress = reduceSubagentProgress( + null, + [ + update({ + activityEventId: 'activity-2', + activitySequence: 2, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'second' }] } }, + }), + update({ + activityEventId: 'activity-1', + activitySequence: 1, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'first ' }] } }, + }), + ], + 'detached', + false, + ); + + expect(progress?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'first second' }]); + expect(progress?.lastActivitySequence).toBe(2); + }); + + it('rejects older overlap events and duplicates beyond the replay-key window', () => { + const initial = reduceSubagentProgress( + null, + [ + update({ + activityEventId: 'activity-300', + activitySequence: 300, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'latest' }] } }, + }), + ], + 'detached', + false, + ); + const delayed = reduceSubagentProgress(initial, [ + update({ + activityEventId: 'activity-1', + activitySequence: 1, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'old' }] } }, + }), + update({ + activityEventId: 'activity-300', + activitySequence: 300, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'duplicate' }] } }, + }), + ]); + + expect(delayed).toBe(initial); + expect(delayed?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'latest' }]); + }); + + it('buffers a detached frame until a lagging parent delivers the missing sequence', () => { + const detached = reduceSubagentProgress( + null, + [ + update({ + activityEventId: 'activity-1', + activitySequence: 1, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'second' }] } }, + }), + ], + 'detached', + true, + ); + expect(detached?.contentParts).toEqual([]); + expect(detached?.pendingSequencedEvents).toHaveLength(1); + + const ordered = reduceSubagentProgress(detached, [ + update({ + activityEventId: 'activity-0', + activitySequence: 0, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'first ' }] } }, + }), + ]); + + expect(ordered?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'first second' }]); + expect(ordered?.pendingSequencedEvents).toBeUndefined(); + expect(ordered?.lastActivitySequence).toBe(1); + expect(ordered?.coverage).toBe('complete'); + }); + + it('uses parent stream closure as the fence for a detached suffix with no earlier frame', () => { + const waiting = reduceSubagentProgress( + null, + [ + update({ + activityEventId: 'activity-5', + activitySequence: 5, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'suffix' }] } }, + }), + ], + 'detached', + true, + ); + + const closed = closeParentSubagentProgress(waiting); + + expect(closed?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'suffix' }]); + expect(closed?.pendingSequencedEvents).toBeUndefined(); + expect(closed?.lastActivitySequence).toBe(5); + + const afterMissedFrames = reduceSubagentProgress( + closed, + [ + update({ + activityEventId: 'activity-8', + activitySequence: 8, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: ' resumed' }] } }, + }), + ], + 'detached', + false, + ); + expect(afterMissedFrames?.contentParts).toEqual([ + { type: ContentTypes.TEXT, text: 'suffix resumed' }, + ]); + expect(afterMissedFrames?.lastActivitySequence).toBe(8); + }); + + it('bounds future sequence buffering while an earlier parent frame is missing', () => { + const waiting = reduceSubagentProgress( + null, + Array.from({ length: 140 }, (_, index) => + update({ + activityEventId: `activity-${index + 1}`, + activitySequence: index + 1, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'x'.repeat(2048) }] } }, + }), + ), + 'detached', + true, + ); + + expect(waiting?.pendingSequencedEvents?.length).toBeLessThanOrEqual(100); + expect( + new TextEncoder().encode(JSON.stringify(waiting?.pendingSequencedEvents)).byteLength, + ).toBeLessThanOrEqual(128 * 1024); + }); + + it('accepts the missing expected frame even when the future-frame buffer is full', () => { + const waiting = reduceSubagentProgress( + null, + Array.from({ length: 100 }, (_, index) => + update({ + activityEventId: `activity-${index + 1}`, + activitySequence: index + 1, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'x' }] } }, + }), + ), + 'detached', + true, + ); + + expect(waiting?.pendingSequencedEvents).toHaveLength(100); + + const ordered = reduceSubagentProgress(waiting, [ + update({ + activityEventId: 'activity-0', + activitySequence: 0, + data: { delta: { content: [{ type: ContentTypes.TEXT, text: 'first-' }] } }, + }), + ]); + + expect(ordered?.contentParts).toEqual([ + { type: ContentTypes.TEXT, text: `first-${'x'.repeat(100)}` }, + ]); + expect(ordered?.pendingSequencedEvents).toBeUndefined(); + expect(ordered?.lastActivitySequence).toBe(100); + expect(ordered?.coverage).toBe('complete'); + }); + + it('preserves legacy unsequenced foreground updates', () => { + const progress = reduceSubagentProgress(null, [ + update({ activityEventId: undefined, activitySequence: undefined }), + update({ activityEventId: undefined, activitySequence: undefined }), + ]); + + expect(progress?.contentParts).toEqual([{ type: ContentTypes.TEXT, text: 'Working.Working.' }]); + expect(progress?.lastActivitySequence).toBeUndefined(); + }); + + it('preserves a reasoning activity marker without retaining private reasoning text', () => { + const progress = reduceSubagentProgress( + null, + [ + update({ + activitySequence: 0, + phase: 'reasoning_delta', + data: { delta: { content: [{ type: ContentTypes.THINK, think: 'private' }] } }, + label: 'Reasoning', + }), + ], + 'detached', + false, + ); + + expect(progress?.contentParts).toEqual([{ type: ContentTypes.THINK, think: '…' }]); + expect(progress?.tickerState.lines).toEqual([ + expect.objectContaining({ kind: 'reasoning', body: '…' }), + ]); + expect(JSON.stringify(progress)).not.toContain('private'); + }); + + it('preserves visible reasoning on the authoritative parent delivery path', () => { + const progress = reduceSubagentProgress(null, [ + update({ + activitySequence: 0, + phase: 'reasoning_delta', + data: { delta: { content: [{ type: ContentTypes.THINK, think: 'Visible reasoning' }] } }, + label: 'Reasoning', + }), + ]); + + expect(progress?.contentParts).toEqual([ + { type: ContentTypes.THINK, think: 'Visible reasoning' }, + ]); + }); + + it('bounds accumulated live text to the durable activity byte budget', () => { + const progress = reduceSubagentProgress(null, [ + update({ + activityEventId: 'large-activity', + data: { delta: { content: [{ type: 'text', text: 'x'.repeat(96 * 1024) }] } }, + }), + ]); + + expect( + new TextEncoder().encode(JSON.stringify(progress?.contentParts)).byteLength, + ).toBeLessThanOrEqual(64 * 1024); + expect(progress?.contentParts[0]).toEqual(expect.objectContaining({ type: ContentTypes.TEXT })); + }); + + it('retains an encoded-byte-bounded singleton containing escaped text', () => { + const progress = reduceSubagentProgress(null, [ + update({ + activityEventId: 'escaped-activity', + data: { delta: { content: [{ type: 'text', text: '\\"'.repeat(48 * 1024) }] } }, + }), + ]); + + expect(progress?.contentParts).toHaveLength(1); + expect(progress?.contentParts[0]).toEqual(expect.objectContaining({ type: ContentTypes.TEXT })); + expect( + new TextEncoder().encode(JSON.stringify(progress?.contentParts)).byteLength, + ).toBeLessThanOrEqual(64 * 1024); + }); + + it('retains an encoded-byte-bounded singleton tool projection', () => { + const progress = reduceSubagentProgress(null, [ + update({ + activityEventId: 'escaped-tool-start', + phase: 'run_step', + data: { + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'tool', name: 'search', args: '\\"'.repeat(48 * 1024) }], + }, + }, + }), + update({ + activityEventId: 'escaped-tool-complete', + phase: 'run_step_completed', + data: { + result: { + type: 'tool_call', + tool_call: { + id: 'tool', + name: 'search', + output: '\\\\'.repeat(48 * 1024), + progress: 1, + }, + }, + }, + }), + ]); + + expect(progress?.contentParts).toHaveLength(1); + expect(progress?.contentParts[0]).toEqual( + expect.objectContaining({ type: ContentTypes.TOOL_CALL }), + ); + expect( + new TextEncoder().encode(JSON.stringify(progress?.contentParts)).byteLength, + ).toBeLessThanOrEqual(64 * 1024); + }); + + it('keeps only the newest bounded activity and continues folding afterward', () => { + const toolEvents = Array.from({ length: 120 }, (_, index) => + update({ + activityEventId: `tool-${index}`, + phase: 'run_step', + data: { + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: `call-${index}`, name: 'search', args: { index } }], + }, + }, + }), + ); + const bounded = reduceSubagentProgress(null, toolEvents); + const continued = reduceSubagentProgress(bounded, [ + update({ + activityEventId: 'after-bound', + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'Final answer.' }] } }, + }), + ]); + + expect(bounded?.contentParts).toHaveLength(100); + expect(continued?.contentParts).toHaveLength(100); + expect(continued?.contentParts.at(-1)).toEqual({ + type: ContentTypes.TEXT, + text: 'Final answer.', + }); + expect(continued?.tickerState.lines.length).toBeLessThanOrEqual(100); + }); +}); diff --git a/client/src/store/subagents.ts b/client/src/store/subagents.ts index 8f6d93ef8f..897bfb9260 100644 --- a/client/src/store/subagents.ts +++ b/client/src/store/subagents.ts @@ -1,14 +1,22 @@ import { atom, atomFamily } from 'recoil'; +import { ContentTypes } from 'librechat-data-provider'; import type { PartMetadata, SubagentUpdatePhase, TMessageContentParts, + SubagentUpdateEvent, } from 'librechat-data-provider'; import type { SubagentAggregatorState, SubagentContentPart, SubagentTickerState, } from '~/utils/subagentContent'; +import { + foldSubagentEvent, + foldSubagentEventIntoTicker, + initSubagentAggregatorState, + initSubagentTickerState, +} from '~/utils/subagentContent'; /** * Progress bucket captured per subagent tool call. Populated as @@ -19,8 +27,8 @@ import type { * Both the panel content and the ticker are aggregated *incrementally* * into the atom as each envelope arrives — the atom never keeps the raw * event array. A long-running subagent can emit thousands of deltas - * without the state growing past what its structural output (N text - * runs + M tool calls + a bounded tail preview) needs. + * without retaining the raw event stream. The folded activity is also + * capped by item count and encoded size to match the durable public view. */ export interface SubagentProgress { /** Child run id from the SDK — unique per spawn; one tool_call may only have one. */ @@ -42,8 +50,203 @@ export interface SubagentProgress { status: SubagentUpdatePhase; /** Convenience: last event's `label` for quick ticker display. */ latestLabel?: string; + /** Bounded replay fence for events that overlap parent and detached SSE delivery. */ + recentEventKeys?: string[]; + /** Highest host sequence folded for this child run. Older overlap frames are ignored. */ + lastActivitySequence?: number; + /** Bounded future frames waiting for an earlier sequence at the parent/detached handoff. */ + pendingSequencedEvents?: SubagentUpdateEvent[]; + /** Whether the folded events cover the run from its beginning or only the + * forward-only suffix observed after opening a detached task stream. */ + coverage?: 'complete' | 'suffix'; } +const MAX_RECENT_EVENT_KEYS = 256; +const MAX_PENDING_SEQUENCE_EVENTS = 100; +const MAX_PENDING_SEQUENCE_BYTES = 128 * 1024; +const MAX_LIVE_ACTIVITY_ITEMS = 100; +const MAX_LIVE_ACTIVITY_BYTES = 64 * 1024; +const MAX_SINGLE_ACTIVITY_ENCODED_BYTES = MAX_LIVE_ACTIVITY_BYTES - 2; +const MAX_SINGLE_ACTIVITY_TEXT_BYTES = 60 * 1024; +const REDACTED_REASONING_MARKER = '…'; + +const encodedBytes = (value: unknown): number => + new TextEncoder().encode(JSON.stringify(value)).byteLength; + +const truncateUtf8 = (value: string, maxBytes: number, keepTail = false): string => { + if (new TextEncoder().encode(value).byteLength <= maxBytes) return value; + const chars = [...value]; + let low = 0; + let high = chars.length; + while (low < high) { + const mid = Math.ceil((low + high) / 2); + const candidate = keepTail ? chars.slice(-mid).join('') : chars.slice(0, mid).join(''); + if (new TextEncoder().encode(candidate).byteLength <= maxBytes) low = mid; + else high = mid - 1; + } + return keepTail ? chars.slice(-low).join('') : chars.slice(0, low).join(''); +}; + +const fitStringField = ( + value: string, + candidate: (bounded: string) => T, + maxBytes: number, + keepTail = false, +): T => { + const chars = [...value]; + let low = 0; + let high = chars.length; + let result = candidate(''); + while (low < high) { + const mid = Math.ceil((low + high) / 2); + const bounded = keepTail ? chars.slice(-mid).join('') : chars.slice(0, mid).join(''); + const next = candidate(bounded); + if (encodedBytes(next) <= maxBytes) { + low = mid; + result = next; + } else { + high = mid - 1; + } + } + return result; +}; + +const boundSingletonPart = (part: SubagentContentPart): SubagentContentPart => { + if (part.type === ContentTypes.TEXT) { + const rawBounded = truncateUtf8(part.text, MAX_SINGLE_ACTIVITY_TEXT_BYTES, true); + return fitStringField( + rawBounded, + (text) => ({ ...part, text }), + MAX_SINGLE_ACTIVITY_ENCODED_BYTES, + true, + ); + } + if (part.type === ContentTypes.THINK) { + const rawBounded = truncateUtf8(part.think, MAX_SINGLE_ACTIVITY_TEXT_BYTES, true); + return fitStringField( + rawBounded, + (think) => ({ ...part, think }), + MAX_SINGLE_ACTIVITY_ENCODED_BYTES, + true, + ); + } + + let bounded: SubagentContentPart = { + ...part, + tool_call: { + ...part.tool_call, + args: truncateUtf8(part.tool_call.args, 24 * 1024), + ...(part.tool_call.output == null + ? {} + : { output: truncateUtf8(part.tool_call.output, 24 * 1024, true) }), + }, + }; + if (encodedBytes(bounded) <= MAX_SINGLE_ACTIVITY_ENCODED_BYTES) return bounded; + + const fitToolField = (field: 'output' | 'args' | 'name' | 'id' | 'type', keepTail = false) => { + if (bounded.type !== ContentTypes.TOOL_CALL) return; + const current = bounded; + const value = current.tool_call[field]; + if (typeof value !== 'string') return; + bounded = fitStringField( + value, + (nextValue) => ({ + ...current, + tool_call: { ...current.tool_call, [field]: nextValue }, + }), + MAX_SINGLE_ACTIVITY_ENCODED_BYTES, + keepTail, + ) as SubagentContentPart; + }; + fitToolField('output', true); + if (encodedBytes(bounded) > MAX_SINGLE_ACTIVITY_ENCODED_BYTES) fitToolField('args'); + if (encodedBytes(bounded) > MAX_SINGLE_ACTIVITY_ENCODED_BYTES) fitToolField('name'); + if (encodedBytes(bounded) > MAX_SINGLE_ACTIVITY_ENCODED_BYTES) fitToolField('id'); + if (encodedBytes(bounded) > MAX_SINGLE_ACTIVITY_ENCODED_BYTES) fitToolField('type'); + return bounded; +}; + +const boundContentParts = ( + parts: SubagentContentPart[], + state: SubagentAggregatorState, +): { parts: SubagentContentPart[]; state: SubagentAggregatorState } => { + const start = Math.max(0, parts.length - MAX_LIVE_ACTIVITY_ITEMS); + let offset = parts.length; + let totalBytes = 2; + let bounded: SubagentContentPart[] = []; + for (let index = parts.length - 1; index >= start; index -= 1) { + const partBytes = encodedBytes(parts[index]); + const separatorBytes = bounded.length === 0 ? 0 : 1; + if (totalBytes + separatorBytes + partBytes > MAX_LIVE_ACTIVITY_BYTES) { + if (bounded.length === 0) { + bounded = [boundSingletonPart(parts[index])]; + offset = index; + } + break; + } + bounded.unshift(parts[index]); + offset = index; + totalBytes += separatorBytes + partBytes; + } + if (encodedBytes(bounded) > MAX_LIVE_ACTIVITY_BYTES) { + bounded = []; + offset = parts.length; + } + const rebase = (index: number | null): number | null => + index != null && index >= offset && index - offset < bounded.length ? index - offset : null; + const toolCallIndexById = Object.fromEntries( + bounded.flatMap((part, index) => + part.type === ContentTypes.TOOL_CALL ? [[part.tool_call.id, index]] : [], + ), + ); + return { + parts: bounded, + state: { + openTextIdx: rebase(state.openTextIdx), + openThinkIdx: rebase(state.openThinkIdx), + toolCallIndexById, + }, + }; +}; + +const boundTickerState = (state: SubagentTickerState): SubagentTickerState => { + const start = Math.max(0, state.lines.length - MAX_LIVE_ACTIVITY_ITEMS); + let offset = state.lines.length; + let totalBytes = 2; + const lines = [] as SubagentTickerState['lines']; + for (let index = state.lines.length - 1; index >= start; index -= 1) { + const lineBytes = encodedBytes(state.lines[index]); + const separatorBytes = lines.length === 0 ? 0 : 1; + if (totalBytes + separatorBytes + lineBytes > MAX_LIVE_ACTIVITY_BYTES) break; + lines.unshift(state.lines[index]); + offset = index; + totalBytes += separatorBytes + lineBytes; + } + const rebase = (index: number | null): number | null => + index != null && index >= offset ? index - offset : null; + return { + ...state, + lines, + textLineIdx: rebase(state.textLineIdx), + thinkLineIdx: rebase(state.thinkLineIdx), + }; +}; + +const hashString = (value: string): string => { + let hash = 2166136261; + for (let index = 0; index < value.length; index++) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return (hash >>> 0).toString(36); +}; + +const eventKey = (event: SubagentUpdateEvent): string | undefined => { + const activityEventId = event.activityEventId?.trim(); + if (!activityEventId) return undefined; + return hashString(`${event.subagentRunId}\u0000${activityEventId}`); +}; + /** One child invocation selected for the shared read-only activity panel. */ export type ActiveSubagentPanel = { host: 'conversation' | 'share'; @@ -82,3 +285,237 @@ export const subagentProgressByToolCallId = atomFamily({ + key: 'subagentParentStreamOpenByToolCallId', + default: false, +}); + +/** + * Invocation atoms populated by either the parent generation stream or the selected detached + * task stream. The conversation host drains this registry on navigation so both transports share + * one cleanup boundary instead of leaking detached-only atom-family members for the app lifetime. + */ +const registeredSubagentProgressKeys = new Set(); + +export function registerSubagentProgressKey(key: string): void { + registeredSubagentProgressKeys.add(key); +} + +export function takeRegisteredSubagentProgressKeys(): string[] { + const keys = [...registeredSubagentProgressKeys]; + registeredSubagentProgressKeys.clear(); + return keys; +} + +export function listRegisteredSubagentProgressKeys(): string[] { + return [...registeredSubagentProgressKeys]; +} + +const validActivitySequence = (value: number | undefined): value is number => + Number.isSafeInteger(value) && value != null && value >= 0; + +const foldAcceptedSubagentEvents = ( + previous: SubagentProgress | null, + events: SubagentUpdateEvent[], + source: 'parent' | 'detached', + pendingSequencedEvents: SubagentUpdateEvent[], +): SubagentProgress | null => { + if (events.length === 0) { + if (previous == null) { + const first = pendingSequencedEvents[0]; + if (first == null) return null; + return { + subagentRunId: first.subagentRunId, + subagentType: first.subagentType, + subagentAgentId: first.subagentAgentId, + contentParts: [], + aggregatorState: initSubagentAggregatorState(), + tickerState: initSubagentTickerState(), + status: first.phase, + recentEventKeys: [], + pendingSequencedEvents, + coverage: source === 'detached' ? 'suffix' : 'complete', + }; + } + if ( + (previous.pendingSequencedEvents == null && pendingSequencedEvents.length === 0) || + (previous.pendingSequencedEvents?.length === pendingSequencedEvents.length && + previous.pendingSequencedEvents.every( + (event, index) => event === pendingSequencedEvents[index], + )) + ) { + return previous; + } + return { + ...previous, + ...(pendingSequencedEvents.length === 0 ? {} : { pendingSequencedEvents }), + }; + } + const recentEventKeys = [...(previous?.recentEventKeys ?? [])]; + for (const event of events) { + const key = eventKey(event); + if (key != null) recentEventKeys.push(key); + } + const boundedEventKeys = recentEventKeys.slice(-MAX_RECENT_EVENT_KEYS); + let contentParts = previous?.contentParts ?? []; + let aggregatorState = previous?.aggregatorState ?? initSubagentAggregatorState(); + let tickerState = previous?.tickerState ?? initSubagentTickerState(); + for (const event of events) { + const foldEvent = + event.phase === 'reasoning_delta' && + event.data == null && + aggregatorState.openThinkIdx == null + ? { + ...event, + data: { + delta: { + content: [{ type: ContentTypes.THINK, think: REDACTED_REASONING_MARKER }], + }, + }, + } + : event; + ({ parts: contentParts, state: aggregatorState } = foldSubagentEvent( + contentParts, + aggregatorState, + foldEvent, + )); + tickerState = foldSubagentEventIntoTicker(tickerState, foldEvent); + } + ({ parts: contentParts, state: aggregatorState } = boundContentParts( + contentParts, + aggregatorState, + )); + tickerState = boundTickerState(tickerState); + const last = events[events.length - 1]; + const lastActivitySequence = [...events] + .reverse() + .map((event) => event.activitySequence) + .find(validActivitySequence); + const effectiveActivitySequence = lastActivitySequence ?? previous?.lastActivitySequence; + const acceptedRunStart = events.some((event) => event.activitySequence === 0); + return { + subagentRunId: last.subagentRunId, + subagentType: last.subagentType, + subagentAgentId: last.subagentAgentId ?? previous?.subagentAgentId, + contentParts, + aggregatorState, + tickerState, + status: last.phase, + latestLabel: last.label ?? previous?.latestLabel, + recentEventKeys: boundedEventKeys, + ...(effectiveActivitySequence == null + ? {} + : { lastActivitySequence: effectiveActivitySequence }), + ...(pendingSequencedEvents.length === 0 ? {} : { pendingSequencedEvents }), + coverage: acceptedRunStart + ? 'complete' + : (previous?.coverage ?? (source === 'detached' ? 'suffix' : 'complete')), + }; +}; + +/** Parent SSE close is an ordering fence: all its earlier frames have already been handled. */ +export function closeParentSubagentProgress( + previous: SubagentProgress | null, +): SubagentProgress | null { + if (previous?.pendingSequencedEvents == null || previous.pendingSequencedEvents.length === 0) { + return previous; + } + const pending = [...previous.pendingSequencedEvents].sort( + (left, right) => (left.activitySequence ?? 0) - (right.activitySequence ?? 0), + ); + return foldAcceptedSubagentEvents( + { ...previous, pendingSequencedEvents: undefined }, + pending, + previous.coverage === 'suffix' ? 'detached' : 'parent', + [], + ); +} + +/** Shared reducer for foreground chat SSE and task-scoped detached activity SSE. */ +export function reduceSubagentProgress( + previous: SubagentProgress | null, + events: SubagentUpdateEvent[], + source: 'parent' | 'detached' = 'parent', + waitForEarlierSequences = source === 'parent', +): SubagentProgress | null { + if (events.length === 0) return previous; + const recentEventKeys = [...(previous?.recentEventKeys ?? [])]; + const seen = new Set(recentEventKeys); + const sequenced = events.every( + (event) => Number.isSafeInteger(event.activitySequence) && (event.activitySequence ?? -1) >= 0, + ); + const orderedEvents = sequenced + ? [...events].sort((left, right) => + (left.activitySequence ?? 0) === (right.activitySequence ?? 0) + ? 0 + : (left.activitySequence ?? 0) - (right.activitySequence ?? 0), + ) + : events; + const sameRun = previous?.subagentRunId === orderedEvents[0]?.subagentRunId; + const lastActivitySequence = sameRun ? previous.lastActivitySequence : undefined; + const pending = sameRun ? [...(previous.pendingSequencedEvents ?? [])] : []; + const pendingSequences = new Set( + pending.map((event) => event.activitySequence).filter(validActivitySequence), + ); + const directEvents: SubagentUpdateEvent[] = []; + let expected = lastActivitySequence == null ? 0 : lastActivitySequence + 1; + if (!waitForEarlierSequences && lastActivitySequence == null) { + const firstSequence = [...pending, ...orderedEvents] + .map((event) => event.activitySequence) + .filter(validActivitySequence) + .sort((left, right) => left - right)[0]; + if (firstSequence != null) expected = firstSequence; + } + + const sanitizeSequencedEvent = (event: SubagentUpdateEvent): SubagentUpdateEvent => + source === 'detached' && event.phase === 'reasoning_delta' + ? { ...event, data: undefined } + : event; + const drainPending = () => { + pending.sort((left, right) => (left.activitySequence ?? 0) - (right.activitySequence ?? 0)); + while (pending[0]?.activitySequence === expected) { + const event = pending.shift(); + if (event == null) break; + pendingSequences.delete(expected); + directEvents.push(event); + expected += 1; + } + }; + + drainPending(); + for (const event of orderedEvents) { + const sequence = event.activitySequence; + const key = eventKey(event); + if (key != null && seen.has(key)) continue; + if (validActivitySequence(sequence)) { + if (sequence < expected || pendingSequences.has(sequence)) continue; + const pendingEvent = sanitizeSequencedEvent(event); + if (sequence === expected) { + directEvents.push(pendingEvent); + expected += 1; + drainPending(); + } else if ( + pending.length < MAX_PENDING_SEQUENCE_EVENTS && + encodedBytes([...pending, pendingEvent]) <= MAX_PENDING_SEQUENCE_BYTES + ) { + pending.push(pendingEvent); + pendingSequences.add(sequence); + } + } else { + if (key != null) seen.add(key); + directEvents.push(event); + } + } + drainPending(); + if ( + !waitForEarlierSequences && + pending[0]?.activitySequence != null && + pending[0].activitySequence > expected + ) { + expected = pending[0].activitySequence; + drainPending(); + } + return foldAcceptedSubagentEvents(previous, directEvents, source, pending); +} diff --git a/client/src/utils/__tests__/subagentContent.test.ts b/client/src/utils/__tests__/subagentContent.test.ts index 35ef518e40..d2792e6ab4 100644 --- a/client/src/utils/__tests__/subagentContent.test.ts +++ b/client/src/utils/__tests__/subagentContent.test.ts @@ -269,6 +269,25 @@ describe('buildSubagentTickerLines', () => { expect(lines[0]).toEqual({ kind: 'writing', body: 'Hello world' }); }); + it('retains meaningful text across a whitespace-heavy stream', () => { + const lines = buildSubagentTickerLines([ + makeEvent({ + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'Visible' }] } }, + }), + makeEvent({ + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: ' \n'.repeat(1200) }] } }, + }), + makeEvent({ + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'again' }] } }, + }), + ]); + + expect(lines[0]).toEqual({ kind: 'writing', body: 'Visible again' }); + }); + it('truncates the writing body to the tail when it grows past the cap', () => { const longText = 'x'.repeat(1000); const lines = buildSubagentTickerLines([ diff --git a/client/src/utils/subagentContent.ts b/client/src/utils/subagentContent.ts index 15755ac94b..712e3ee822 100644 --- a/client/src/utils/subagentContent.ts +++ b/client/src/utils/subagentContent.ts @@ -294,8 +294,8 @@ export interface SubagentTickerState { textLineIdx: number | null; /** Index of the in-flight 'reasoning' line. */ thinkLineIdx: number | null; - /** Raw message-delta accumulator — truncated into `writing.body` but - * preserved so subsequent deltas extend the running preview. */ + /** Whitespace-normalized message-delta accumulator. A trailing separator is + * retained so chunk boundaries still render as one word boundary. */ textBuffer: string; thinkBuffer: string; } @@ -317,12 +317,20 @@ export function initSubagentTickerState(): SubagentTickerState { * CSS ellipsis — double-eliding would render a stray dot character * right next to the "Writing:" / "Reasoning:" label. */ const PREVIEW_MAX_CHARS = 300; +const PREVIEW_BUFFER_MAX_CHARS = PREVIEW_MAX_CHARS * 4; const truncatePreview = (input: string): string => { const normalized = input.replace(/\s+/g, ' ').trim(); if (normalized.length <= PREVIEW_MAX_CHARS) return normalized; return normalized.slice(-PREVIEW_MAX_CHARS); }; +const appendPreviewBuffer = (buffer: string, chunk: string): string => { + const normalized = `${buffer}${chunk}`.replace(/\s+/g, ' ').trimStart(); + return normalized.length <= PREVIEW_BUFFER_MAX_CHARS + ? normalized + : normalized.slice(-PREVIEW_BUFFER_MAX_CHARS); +}; + const SNIPPET_MAX_CHARS = 48; /** Short head-truncation for tool args/output — caller labels what each * side is. Whitespace collapsed so multi-line outputs stay one line. */ @@ -396,7 +404,7 @@ export function foldSubagentEventIntoTicker( state.thinkLineIdx != null || state.thinkBuffer ? { ...state, thinkLineIdx: null, thinkBuffer: '' } : state; - const textBuffer = afterClose.textBuffer + chunk; + const textBuffer = appendPreviewBuffer(afterClose.textBuffer, chunk); const body = truncatePreview(textBuffer); const line: SubagentTickerLine = { kind: 'writing', body }; if (afterClose.textLineIdx == null) { @@ -416,7 +424,7 @@ export function foldSubagentEventIntoTicker( state.textLineIdx != null || state.textBuffer ? { ...state, textLineIdx: null, textBuffer: '' } : state; - const thinkBuffer = afterClose.thinkBuffer + chunk; + const thinkBuffer = appendPreviewBuffer(afterClose.thinkBuffer, chunk); const body = truncatePreview(thinkBuffer); const line: SubagentTickerLine = { kind: 'reasoning', body }; if (afterClose.thinkLineIdx == null) { @@ -447,7 +455,7 @@ export function foldSubagentEventIntoTicker( typeof tc?.name === 'string' && tc.name.length > 0, ); if (named.length === 0) return afterClose; - const toolNames = named.map((tc) => tc.name); + const toolNames = named.slice(0, 16).map((tc) => truncateSnippet(tc.name)); const argsSnippet = named.length === 1 ? summarizeArgs(named[0].args) : undefined; const line: SubagentTickerLine = { kind: 'using_tool', @@ -464,7 +472,7 @@ export function foldSubagentEventIntoTicker( const outputSnippet = tc.output != null ? summarizeOutput(tc.output) : undefined; const line: SubagentTickerLine = { kind: 'tool_complete', - toolName: tc.name, + toolName: truncateSnippet(tc.name), ...(outputSnippet ? { outputSnippet } : {}), }; return { ...state, lines: state.lines.concat(line) }; @@ -474,7 +482,7 @@ export function foldSubagentEventIntoTicker( const data = event.data as ErrorData | undefined; const line: SubagentTickerLine = { kind: 'error', - ...(data?.message ? { message: data.message } : {}), + ...(data?.message ? { message: truncatePreview(data.message) } : {}), }; return { ...state, lines: state.lines.concat(line) }; } diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts index 63d2a877a0..869c2890cc 100644 --- a/packages/api/src/agents/index.ts +++ b/packages/api/src/agents/index.ts @@ -33,6 +33,7 @@ export * from './skills'; export * from './phases'; export * from './startup'; export * from './subagentThreads'; +export * from './subagentActivity'; export * from './subagentCompletionWakeup'; export * from './subagentTaskRouting'; export * from './skillConfigurable'; diff --git a/packages/api/src/agents/subagentActivity.spec.ts b/packages/api/src/agents/subagentActivity.spec.ts new file mode 100644 index 0000000000..6ad331e69e --- /dev/null +++ b/packages/api/src/agents/subagentActivity.spec.ts @@ -0,0 +1,599 @@ +import { EventEmitter } from 'node:events'; +import type { IConversation, IMessage } from '@librechat/data-schemas'; +import type { SubagentUpdateEvent } from '@librechat/agents'; +import type { Response } from 'express'; +import type { SubagentActivityEnvelope, SubagentActivityUpdateEvent } from './subagentActivity'; +import type { IEventTransport } from '~/stream/interfaces/IJobStore'; +import type { ServerRequest } from '~/types'; +import { + SubagentActivityStream, + createSubagentActivityStreamHandler, + subagentActivityStreamId, +} from './subagentActivity'; + +class TestTransport implements IEventTransport { + readonly handlers = new Map< + string, + Map< + number, + { + onChunk: (event: unknown) => void; + onDone?: (event: unknown) => void; + onError?: (error: string) => void; + } + > + >(); + + readonly emitted: Array<{ streamId: string; event: unknown }> = []; + readonly completed: Array<{ streamId: string; event: unknown }> = []; + readonly cleaned: string[] = []; + readonly synchronized: string[] = []; + readonly subscribeOptions: unknown[] = []; + readonly closed: Array<{ streamId: string; error: string }> = []; + + demanded = true; + subscriptionReady?: Promise; + private nextSubscriberId = 0; + + subscribe( + streamId: string, + handlers: { + onChunk: (event: unknown) => void; + onDone?: (event: unknown) => void; + onError?: (error: string) => void; + }, + options?: unknown, + ) { + this.subscribeOptions.push(options); + const subscribers = this.handlers.get(streamId) ?? new Map(); + const subscriberId = ++this.nextSubscriberId; + subscribers.set(subscriberId, handlers); + this.handlers.set(streamId, subscribers); + return { + ...(this.subscriptionReady == null ? {} : { ready: this.subscriptionReady }), + syncReorderBuffer: () => { + if (this.handlers.get(streamId) !== subscribers) return; + this.syncReorderBuffer(streamId); + }, + unsubscribe: () => { + subscribers.delete(subscriberId); + if (subscribers.size === 0) this.handlers.delete(streamId); + }, + }; + } + + syncReorderBuffer(streamId: string): void { + this.synchronized.push(streamId); + } + + emitChunk(streamId: string, event: unknown): void { + this.emitted.push({ streamId, event }); + for (const handlers of this.handlers.get(streamId)?.values() ?? []) { + handlers.onChunk(event); + } + } + + emitDone(streamId: string, event: unknown): void { + this.completed.push({ streamId, event }); + for (const handlers of this.handlers.get(streamId)?.values() ?? []) { + handlers.onDone?.(event); + } + } + + emitError(streamId: string, error: string): void { + for (const handlers of this.handlers.get(streamId)?.values() ?? []) { + handlers.onError?.(error); + } + } + + renewDemand(): void { + this.demanded = true; + } + + hasDemand(): boolean { + return this.demanded; + } + + getSubscriberCount(streamId: string): number { + return this.handlers.get(streamId)?.size ?? 0; + } + + isFirstSubscriber(streamId: string): boolean { + return this.getSubscriberCount(streamId) === 1; + } + + onAllSubscribersLeft(): void {} + + cleanup(streamId: string): void { + this.cleaned.push(streamId); + this.handlers.delete(streamId); + } + + getTrackedStreamIds(): string[] { + return [...this.handlers.keys()]; + } + + closeLocalSubscribers(streamId: string, error: string): void { + this.closed.push({ streamId, error }); + const subscribers = this.handlers.get(streamId); + if (subscribers == null) return; + for (const handlers of [...subscribers.values()]) { + handlers.onError?.(error); + } + } + + destroy(): void { + this.handlers.clear(); + } +} + +const update = ( + overrides: Partial = {}, +): SubagentActivityUpdateEvent => ({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [ + { + subagentRunId: 'parent-run', + subagentType: 'parent', + subagentKind: 'agent', + subagentAgentId: 'parent-agent', + parentRunId: 'root-run', + }, + ], + phase: 'message_delta', + data: { delta: 'Working.' }, + label: 'Drafting the report', + timestamp: '2026-08-21T20:00:00.000Z', + ...overrides, +}); + +describe('detached subagent activity stream', () => { + it('uses a stable opaque stream id and forwards the existing update envelope', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const received: unknown[] = []; + const streamId = subagentActivityStreamId('child-thread', 'task-1'); + const subscription = stream.subscribe('child-thread', 'task-1', { + onEvent: (event) => received.push(event), + }); + await subscription.ready; + + await stream.publish( + 'child-thread', + 'task-1', + update({ activityEventId: 'task-1:7', activitySequence: 7 }), + ); + + expect(streamId).toMatch(/^subagent-activity:[A-Za-z0-9_-]{32}$/); + expect(transport.subscribeOptions).toEqual([ + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ]); + expect(transport.synchronized).toEqual([streamId]); + expect(received).toEqual([ + expect.objectContaining({ + event: 'on_subagent_update', + data: expect.objectContaining({ + label: 'Drafting the report', + activityEventId: 'task-1:7', + activitySequence: 7, + }), + }), + ]); + subscription.unsubscribe(); + }); + + it('omits an invalid activity sequence from the public envelope', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + + await stream.publish( + 'child-thread', + 'task-1', + update({ activityEventId: 'task-1:invalid', activitySequence: -1 }), + ); + + expect((transport.emitted[0]?.event as SubagentActivityEnvelope).data).not.toHaveProperty( + 'activitySequence', + ); + }); + + it('does not resynchronize when another local subscriber joins an active stream', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const first = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + await first.ready; + + const second = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + await second.ready; + + expect(transport.subscribeOptions).toEqual([ + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + { deferSequenceDelivery: false, captureSequenceFrontier: false }, + ]); + expect(transport.synchronized).toEqual([subagentActivityStreamId('child-thread', 'task-1')]); + first.unsubscribe(); + second.unsubscribe(); + }); + + it('closes local activity subscribers before HTTP drain', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const onError = jest.fn(); + const streamId = subagentActivityStreamId('child-thread', 'task-1'); + const subscription = stream.subscribe('child-thread', 'task-1', { + onEvent: jest.fn(), + onError, + }); + await subscription.ready; + + stream.prepareForShutdown(); + + expect(transport.closed).toEqual([{ streamId, error: 'Server is shutting down' }]); + expect(onError).toHaveBeenCalledWith('Server is shutting down'); + expect(transport.getSubscriberCount(streamId)).toBe(0); + }); + + it('finishes first-attachment synchronization for a surviving second subscriber', async () => { + const transport = new TestTransport(); + let markReady!: () => void; + transport.subscriptionReady = new Promise((resolve) => (markReady = resolve)); + const stream = new SubagentActivityStream(transport); + const first = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + const second = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + + first.unsubscribe(); + markReady(); + await Promise.all([first.ready, second.ready]); + + expect(transport.getSubscriberCount(subagentActivityStreamId('child-thread', 'task-1'))).toBe( + 1, + ); + expect(transport.synchronized).toEqual([subagentActivityStreamId('child-thread', 'task-1')]); + second.unsubscribe(); + }); + + it('does not let a stale attachment synchronize recreated transport state', async () => { + const transport = new TestTransport(); + let markOldReady!: () => void; + transport.subscriptionReady = new Promise((resolve) => (markOldReady = resolve)); + const stream = new SubagentActivityStream(transport); + const stale = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + stale.unsubscribe(); + await Promise.resolve(); + + transport.subscriptionReady = Promise.resolve(); + const replacement = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + await replacement.ready; + markOldReady(); + await stale.ready; + + expect(transport.synchronized).toEqual([subagentActivityStreamId('child-thread', 'task-1')]); + replacement.unsubscribe(); + }); + + it('publishes only while a panel has renewed live-view demand', async () => { + const transport = new TestTransport(); + transport.demanded = false; + const stream = new SubagentActivityStream(transport); + + await stream.publish('child-thread', 'task-1', update()); + expect(transport.emitted).toHaveLength(0); + + const subscription = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + await subscription.ready; + await stream.publish('child-thread', 'task-1', update()); + + expect(transport.emitted).toHaveLength(1); + subscription.unsubscribe(); + await Promise.resolve(); + expect(transport.cleaned).toEqual([subagentActivityStreamId('child-thread', 'task-1')]); + }); + + it('does not cache a replica-local no-demand observation', async () => { + const transport = new TestTransport(); + transport.demanded = false; + const stream = new SubagentActivityStream(transport); + + await stream.publish('child-thread', 'task-1', update()); + transport.demanded = true; + await stream.publish('child-thread', 'task-1', update()); + + expect(transport.emitted).toHaveLength(1); + }); + + it('evicts cached no-demand state when a task reaches terminal state', async () => { + const transport = new TestTransport(); + transport.demanded = false; + const stream = new SubagentActivityStream(transport); + + await stream.publish('child-thread', 'task-1', update()); + await stream.complete('child-thread', 'task-1', 'completed'); + transport.demanded = true; + await stream.publish('child-thread', 'task-1', update()); + + expect(transport.emitted).toHaveLength(1); + }); + + it('removes local demand state when renewal finishes after disconnect', async () => { + const transport = new TestTransport(); + let markRenewing!: () => void; + let finishRenewal!: () => void; + const renewing = new Promise((resolve) => (markRenewing = resolve)); + transport.renewDemand = jest.fn( + () => + new Promise((resolve) => { + finishRenewal = resolve; + markRenewing(); + }), + ); + const stream = new SubagentActivityStream(transport); + + const subscription = stream.subscribe('child-thread', 'task-1', { onEvent: jest.fn() }); + await renewing; + subscription.unsubscribe(); + finishRenewal(); + await subscription.ready; + transport.demanded = false; + await stream.publish('child-thread', 'task-1', update()); + + expect(transport.emitted).toHaveLength(0); + }); + + it('drops oversized payload data while retaining lifecycle identity and bounds', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + + await stream.publish( + 'child-thread', + 'task-1', + update({ data: { delta: 'x'.repeat(256 * 1024) }, label: 'y'.repeat(4096) }), + ); + + const envelope = transport.emitted[0]?.event as { + data: SubagentUpdateEvent; + }; + expect(envelope.data.data).toBeUndefined(); + expect(envelope.data.label?.length).toBeLessThanOrEqual(512); + expect(Buffer.byteLength(JSON.stringify(envelope), 'utf8')).toBeLessThanOrEqual(64 * 1024); + }); + + it('never transports hidden reasoning text to the detached panel', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + + await stream.publish( + 'child-thread', + 'task-1', + update({ phase: 'reasoning_delta', data: { delta: { content: [{ think: 'secret' }] } } }), + ); + + expect((transport.emitted[0]?.event as SubagentActivityEnvelope).data.data).toBeUndefined(); + }); + + it('delivers terminal state before the subscriber releases its task stream', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const done: unknown[] = []; + stream.subscribe('child-thread', 'task-1', { + onEvent: jest.fn(), + onDone: (event) => done.push(event), + }); + + await stream.complete('child-thread', 'task-1', 'completed'); + + expect(done).toEqual([{ final: true, subagentActivity: true, status: 'completed' }]); + expect(transport.completed).toHaveLength(1); + expect(transport.handlers.size).toBe(0); + await Promise.resolve(); + expect(transport.cleaned).toEqual([subagentActivityStreamId('child-thread', 'task-1')]); + }); +}); + +type StreamResponse = Response & + EventEmitter & { + chunks: string[]; + writableEnded: boolean; + }; + +const response = (): StreamResponse => { + const emitter = new EventEmitter() as StreamResponse; + emitter.chunks = []; + emitter.writableEnded = false; + emitter.status = jest.fn(() => emitter); + emitter.json = jest.fn(() => emitter); + emitter.setHeader = jest.fn(); + emitter.flushHeaders = jest.fn(); + emitter.write = jest.fn((chunk: string) => { + emitter.chunks.push(chunk); + return true; + }); + emitter.end = jest.fn(() => { + emitter.writableEnded = true; + return emitter; + }); + return emitter; +}; + +describe('subagent activity stream authorization', () => { + const parentConversationId = 'parent-conversation'; + const threadId = 'child-thread'; + const taskId = 'task-1'; + const parent = { tenantId: 'tenant-1' } as IConversation; + const child = { + tenantId: 'tenant-1', + subagentThread: { parentConversationId }, + subagentThreadLease: { + token: 'lease', + taskId, + expiresAt: new Date('2099-01-01T00:00:00.000Z'), + }, + } as unknown as IConversation; + const request = () => { + const req = new EventEmitter() as ServerRequest & EventEmitter; + req.params = { parentConversationId, threadId, taskId }; + req.user = { id: 'user-1', tenantId: 'tenant-1' } as ServerRequest['user']; + return req; + }; + + it('streams only the exact active task through its owning parent', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const handler = createSubagentActivityStreamHandler( + { + getConvoOwnership: jest.fn().mockResolvedValue(parent), + getSubagentThreadForParent: jest.fn().mockResolvedValue(child), + getMessages: jest.fn().mockResolvedValue([]), + }, + stream, + ); + const req = request(); + const res = response(); + + await handler(req, res); + await stream.publish(threadId, taskId, update()); + + expect(res.status).not.toHaveBeenCalled(); + expect(res.chunks.join('')).toContain('"event":"on_subagent_update"'); + expect(res.chunks.join('')).toContain('Drafting the report'); + res.emit('close'); + }); + + it('returns the same 404 for a mismatched task without subscribing', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const handler = createSubagentActivityStreamHandler( + { + getConvoOwnership: jest.fn().mockResolvedValue(parent), + getSubagentThreadForParent: jest.fn().mockResolvedValue(child), + getMessages: jest.fn().mockResolvedValue([]), + }, + stream, + ); + const req = request(); + (req.params as Record).taskId = 'different-task'; + const res = response(); + + await handler(req, res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(transport.handlers.size).toBe(0); + }); + + it('does not subscribe when the client disconnects during authorization', async () => { + let resolveParent!: (value: IConversation) => void; + let resolveChild!: (value: IConversation) => void; + const stream = { subscribe: jest.fn() }; + const handler = createSubagentActivityStreamHandler( + { + getConvoOwnership: jest.fn( + () => new Promise((resolve) => (resolveParent = resolve)), + ), + getSubagentThreadForParent: jest.fn( + () => new Promise((resolve) => (resolveChild = resolve)), + ), + getMessages: jest.fn().mockResolvedValue([]), + }, + stream, + ); + const req = request(); + const res = response(); + + const pending = handler(req, res); + res.emit('close'); + resolveParent(parent); + resolveChild(child); + await pending; + + expect(stream.subscribe).not.toHaveBeenCalled(); + expect(res.flushHeaders).not.toHaveBeenCalled(); + }); + + it('ends the SSE and releases its subscription when readiness fails', async () => { + const transport = new TestTransport(); + let rejectReady!: (error: Error) => void; + transport.subscriptionReady = new Promise((_resolve, reject) => { + rejectReady = reject; + }); + const stream = new SubagentActivityStream(transport); + const handler = createSubagentActivityStreamHandler( + { + getConvoOwnership: jest.fn().mockResolvedValue(parent), + getSubagentThreadForParent: jest.fn().mockResolvedValue(child), + getMessages: jest.fn().mockResolvedValue([]), + }, + stream, + ); + const res = response(); + const pending = handler(request(), res); + while (transport.handlers.size === 0) { + await Promise.resolve(); + } + Object.defineProperty(res, 'headersSent', { value: true, configurable: true }); + + rejectReady(new Error('Redis subscription unavailable')); + await pending; + + expect(res.chunks.join('')).toContain('Subagent activity stream unavailable'); + expect(res.end).toHaveBeenCalledTimes(1); + expect(transport.handlers.size).toBe(0); + }); + + it('closes with durable terminal state when completion races stream readiness', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const handler = createSubagentActivityStreamHandler( + { + getConvoOwnership: jest.fn().mockResolvedValue(parent), + getSubagentThreadForParent: jest.fn().mockResolvedValue(child), + getMessages: jest.fn().mockResolvedValue([ + { + messageId: `${taskId}:assistant`, + subagentTask: { status: 'completed' }, + } as IMessage, + ]), + }, + stream, + ); + const res = response(); + + await handler(request(), res); + + expect(res.chunks.join('')).toContain('"final":true'); + expect(res.chunks.join('')).toContain('"status":"completed"'); + expect(res.end).toHaveBeenCalledTimes(1); + expect(transport.handlers.size).toBe(0); + }); + + it('closes a slow SSE consumer instead of buffering later activity', async () => { + const transport = new TestTransport(); + const stream = new SubagentActivityStream(transport); + const handler = createSubagentActivityStreamHandler( + { + getConvoOwnership: jest.fn().mockResolvedValue(parent), + getSubagentThreadForParent: jest.fn().mockResolvedValue(child), + getMessages: jest.fn().mockResolvedValue([]), + }, + stream, + ); + const res = response(); + let writes = 0; + (res.write as jest.Mock).mockImplementation((chunk: string) => { + res.chunks.push(chunk); + writes += 1; + return writes === 1; + }); + + await handler(request(), res); + await stream.publish(threadId, taskId, update()); + + expect(res.end).toHaveBeenCalledTimes(1); + expect(transport.handlers.size).toBe(0); + }); +}); diff --git a/packages/api/src/agents/subagentActivity.ts b/packages/api/src/agents/subagentActivity.ts new file mode 100644 index 0000000000..f91ec2e700 --- /dev/null +++ b/packages/api/src/agents/subagentActivity.ts @@ -0,0 +1,510 @@ +import { createHash } from 'node:crypto'; +import { logger } from '@librechat/data-schemas'; +import type { ConversationMethods, MessageMethods } from '@librechat/data-schemas'; +import type { SubagentUpdateEvent } from '@librechat/agents'; +import type { Response } from 'express'; +import type { IEventTransport } from '~/stream/interfaces/IJobStore'; +import type { ServerRequest } from '~/types'; +import { emitObservedChunk } from '~/stream/internal/chunkPublication'; + +const STREAM_PREFIX = 'subagent-activity:'; +const MAX_ID_BYTES = 512; +const MAX_LABEL_BYTES = 512; +const MAX_ANCESTRY_ENTRIES = 16; +const MAX_EVENT_BYTES = 64 * 1024; +const HEARTBEAT_MS = 15_000; +const DEMAND_TTL_MS = 30_000; +const DEMAND_HEARTBEAT_MS = 10_000; +const DEMAND_CACHE_MS = 250; +const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down'; + +export type SubagentActivityTerminalStatus = 'completed' | 'failed' | 'cancelled'; + +export type SubagentActivityUpdateEvent = SubagentUpdateEvent & { + /** Host-assigned identity shared by parent and detached delivery paths. */ + activityEventId?: string; + /** Host-assigned monotonic sequence shared by parent and detached delivery paths. */ + activitySequence?: number; +}; + +export type SubagentActivityEnvelope = { + event: 'on_subagent_update'; + data: SubagentActivityUpdateEvent; +}; + +export type SubagentActivitySubscription = { + unsubscribe: () => void; + ready?: Promise; +}; + +export type SubagentActivitySubscriber = { + onEvent: (event: SubagentActivityEnvelope) => void; + onDone?: (event: { + final: true; + subagentActivity: true; + status: SubagentActivityTerminalStatus; + }) => void; + onError?: (error: string) => void; +}; + +type SubagentActivityStreamDependencies = Pick< + ConversationMethods, + 'getConvoOwnership' | 'getSubagentThreadForParent' +> & + Pick; + +type SubagentActivityStreamParams = { + parentConversationId?: string; + threadId?: string; + taskId?: string; +}; + +const validId = (value: string | undefined): value is string => + value != null && value.trim() !== '' && Buffer.byteLength(value, 'utf8') <= MAX_ID_BYTES; + +const boundedString = (value: string | undefined, maxBytes = MAX_ID_BYTES): string | undefined => { + if (value == null) return undefined; + if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value; + let end = Math.min(value.length, maxBytes); + while (end > 0 && Buffer.byteLength(value.slice(0, end), 'utf8') > maxBytes) end -= 1; + return value.slice(0, end); +}; + +const boundedData = (data: unknown, budget: number): unknown => { + if (data == null) return undefined; + try { + return Buffer.byteLength(JSON.stringify(data), 'utf8') <= budget ? data : undefined; + } catch { + return undefined; + } +}; + +export const boundSubagentActivityUpdate = ( + event: SubagentActivityUpdateEvent, +): SubagentActivityUpdateEvent => { + let base: SubagentActivityUpdateEvent = { + runId: boundedString(event.runId) ?? '', + parentRunId: boundedString(event.parentRunId) ?? '', + subagentRunId: boundedString(event.subagentRunId) ?? '', + ...(boundedString(event.activityEventId) == null + ? {} + : { activityEventId: boundedString(event.activityEventId) }), + ...(Number.isSafeInteger(event.activitySequence) && (event.activitySequence ?? -1) >= 0 + ? { activitySequence: event.activitySequence } + : {}), + subagentType: boundedString(event.subagentType) ?? '', + subagentKind: event.subagentKind, + subagentAgentId: boundedString(event.subagentAgentId) ?? '', + ...(boundedString(event.memberAgentId) == null + ? {} + : { memberAgentId: boundedString(event.memberAgentId) }), + ...(boundedString(event.parentAgentId) == null + ? {} + : { parentAgentId: boundedString(event.parentAgentId) }), + ...(boundedString(event.parentToolCallId) == null + ? {} + : { parentToolCallId: boundedString(event.parentToolCallId) }), + depth: event.depth, + ancestry: (event.ancestry ?? []).slice(0, MAX_ANCESTRY_ENTRIES).map((entry) => ({ + subagentRunId: boundedString(entry.subagentRunId) ?? '', + subagentType: boundedString(entry.subagentType) ?? '', + subagentKind: entry.subagentKind, + subagentAgentId: boundedString(entry.subagentAgentId) ?? '', + parentRunId: boundedString(entry.parentRunId) ?? '', + ...(boundedString(entry.parentAgentId) == null + ? {} + : { parentAgentId: boundedString(entry.parentAgentId) }), + ...(boundedString(entry.parentToolCallId) == null + ? {} + : { parentToolCallId: boundedString(entry.parentToolCallId) }), + })), + phase: event.phase, + ...(boundedString(event.label, MAX_LABEL_BYTES) == null + ? {} + : { label: boundedString(event.label, MAX_LABEL_BYTES) }), + timestamp: boundedString(event.timestamp) ?? new Date().toISOString(), + }; + let baseBytes = Buffer.byteLength( + JSON.stringify({ event: 'on_subagent_update', data: base }), + 'utf8', + ); + if (baseBytes > MAX_EVENT_BYTES) { + base = { ...base, ancestry: [] }; + baseBytes = Buffer.byteLength( + JSON.stringify({ event: 'on_subagent_update', data: base }), + 'utf8', + ); + } + /** Detached durable views expose only a reasoning marker. Keep that same boundary + * on the live path rather than transporting hidden reasoning text to the browser. */ + const data = + event.phase === 'reasoning_delta' + ? undefined + : boundedData(event.data, Math.max(0, MAX_EVENT_BYTES - baseBytes - 32)); + return data == null ? base : { ...base, data }; +}; + +const isTerminalEvent = ( + value: unknown, +): value is { + final: true; + subagentActivity: true; + status: SubagentActivityTerminalStatus; +} => { + if (value == null || typeof value !== 'object') return false; + const event = value as { + final?: unknown; + subagentActivity?: unknown; + status?: unknown; + }; + return ( + event.final === true && + event.subagentActivity === true && + (event.status === 'completed' || event.status === 'failed' || event.status === 'cancelled') + ); +}; + +const isActivityEnvelope = (value: unknown): value is SubagentActivityEnvelope => { + if (value == null || typeof value !== 'object') return false; + const envelope = value as { event?: unknown; data?: unknown }; + return ( + envelope.event === 'on_subagent_update' && + envelope.data != null && + typeof envelope.data === 'object' + ); +}; + +export const subagentActivityStreamId = (threadId: string, taskId: string): string => + `${STREAM_PREFIX}${createHash('sha256') + .update(`${threadId}\u0000${taskId}`) + .digest('base64url') + .slice(0, 32)}`; + +/** Task-scoped live activity over the same in-memory/Redis transports used by generation SSE. */ +export class SubagentActivityStream { + private readonly demandCache = new Map(); + + constructor(private readonly transport: IEventTransport) {} + + private async isDemanded(streamId: string): Promise { + if (this.transport.hasDemand == null) return true; + const cached = this.demandCache.get(streamId); + if (cached != null && cached.expiresAt > Date.now()) return cached.demanded; + const demanded = await this.transport.hasDemand(streamId); + /** A negative observation is replica-local and can become stale as soon as a panel on + * another owner renews the shared lease. Cache only positive demand so attachment never + * creates a forward-only delivery hole on a remote producer. */ + if (demanded) { + this.demandCache.set(streamId, { demanded: true, expiresAt: Date.now() + DEMAND_CACHE_MS }); + } else { + this.demandCache.delete(streamId); + } + return demanded; + } + + private async renewDemand(streamId: string, isActive = () => true): Promise { + await this.transport.renewDemand?.(streamId, DEMAND_TTL_MS); + if (!isActive()) { + this.demandCache.delete(streamId); + return; + } + this.demandCache.set(streamId, { demanded: true, expiresAt: Date.now() + DEMAND_CACHE_MS }); + } + + async publish( + threadId: string, + taskId: string, + event: SubagentActivityUpdateEvent, + ): Promise { + const streamId = subagentActivityStreamId(threadId, taskId); + if (!(await this.isDemanded(streamId))) return; + const envelope: SubagentActivityEnvelope = { + event: 'on_subagent_update', + data: boundSubagentActivityUpdate(event), + }; + await emitObservedChunk(this.transport, streamId, envelope); + } + + subscribe( + threadId: string, + taskId: string, + subscriber: SubagentActivitySubscriber, + ): SubagentActivitySubscription { + const streamId = subagentActivityStreamId(threadId, taskId); + let unsubscribe = (): void => undefined; + const cleanupIfIdle = (): void => { + queueMicrotask(() => { + if (this.transport.getSubscriberCount(streamId) > 0) return; + this.transport.cleanup(streamId); + this.demandCache.delete(streamId); + }); + }; + /** subscribe() registers synchronously. Sampling zero before it distinguishes a fresh + * local attachment without moving an already-active shared reorder frontier. */ + const synchronizeAttachment = this.transport.getSubscriberCount(streamId) === 0; + const subscription = this.transport.subscribe( + streamId, + { + onChunk: (event) => { + if (isActivityEnvelope(event)) subscriber.onEvent(event); + }, + onDone: (event) => { + if (!isTerminalEvent(event)) return; + try { + subscriber.onDone?.(event); + } finally { + unsubscribe(); + } + }, + onError: (error) => { + try { + subscriber.onError?.(error); + } finally { + unsubscribe(); + } + }, + }, + { + deferSequenceDelivery: synchronizeAttachment, + captureSequenceFrontier: synchronizeAttachment, + }, + ); + let closed = false; + let demandHeartbeat: ReturnType | undefined; + unsubscribe = () => { + if (closed) return; + closed = true; + if (demandHeartbeat != null) clearInterval(demandHeartbeat); + subscription.unsubscribe(); + cleanupIfIdle(); + }; + const ready = Promise.resolve(subscription.ready).then(async () => { + if (synchronizeAttachment) { + /** The attachment that deferred the shared buffer owns synchronization even if + * its panel closes meanwhile; a surviving local subscriber still needs release. */ + await subscription.syncReorderBuffer?.(); + } + if (closed) return; + await this.renewDemand(streamId, () => !closed); + if (closed) { + this.demandCache.delete(streamId); + return; + } + demandHeartbeat = setInterval(() => { + void this.renewDemand(streamId, () => !closed).catch(() => undefined); + }, DEMAND_HEARTBEAT_MS); + demandHeartbeat.unref?.(); + }); + return { unsubscribe, ready }; + } + + async complete( + threadId: string, + taskId: string, + status: SubagentActivityTerminalStatus, + ): Promise { + const streamId = subagentActivityStreamId(threadId, taskId); + this.demandCache.delete(streamId); + try { + if (!(await this.isDemanded(streamId))) return; + await this.transport.emitDone(streamId, { + final: true, + subagentActivity: true, + status, + }); + } finally { + this.demandCache.delete(streamId); + } + } + + /** Close this process's SSE responses before HTTP drain. Durable child execution and + * cross-replica activity remain untouched; clients reconnect to another live owner. */ + prepareForShutdown(): void { + for (const streamId of this.transport.getTrackedStreamIds()) { + this.transport.closeLocalSubscribers?.(streamId, SHUTDOWN_SUBSCRIBER_ERROR); + } + } + + destroy(): void { + this.demandCache.clear(); + this.transport.destroy(); + } +} + +const terminalStatus = (status: string | undefined): SubagentActivityTerminalStatus | undefined => { + switch (status) { + case 'completed': + return 'completed'; + case 'error': + return 'failed'; + case 'cancelled': + return 'cancelled'; + default: + return undefined; + } +}; + +const terminalTaskStatus = async ( + deps: Pick, + userId: string, + threadId: string, + taskId: string, + tenantId?: string, +): Promise => { + const messages = await deps.getMessages( + { + user: userId, + conversationId: threadId, + messageId: `${taskId}:assistant`, + ...(tenantId == null ? { tenantId: { $exists: false } } : { tenantId }), + }, + 'messageId +subagentTask', + { limit: 1 }, + ); + return terminalStatus(messages[0]?.subagentTask?.status); +}; + +const notFound = (res: Response): void => { + res.status(404).json({ error: 'Conversation not found' }); +}; + +const writeSse = (res: Response, value: unknown): boolean => + !res.writableEnded && res.write(`data: ${JSON.stringify(value)}\n\n`); + +/** Streams one active child task after the same parent/tenant authorization as its durable view. */ +export function createSubagentActivityStreamHandler( + deps: SubagentActivityStreamDependencies, + stream: Pick, +) { + return async (req: ServerRequest, res: Response): Promise => { + const userId = req.user?.id; + const tenantId = req.user?.tenantId || undefined; + const { parentConversationId, threadId, taskId } = req.params as SubagentActivityStreamParams; + if ( + !userId || + !validId(parentConversationId) || + !validId(threadId) || + !validId(taskId) || + parentConversationId === threadId + ) { + notFound(res); + return; + } + + let closed = req.destroyed || res.destroyed; + let heartbeat: ReturnType | undefined; + let subscription: SubagentActivitySubscription | undefined; + const dispose = () => { + if (heartbeat != null) clearInterval(heartbeat); + subscription?.unsubscribe(); + }; + const close = () => { + closed = true; + dispose(); + }; + req.once('aborted', close); + res.once('close', close); + + try { + const [parent, child] = await Promise.all([ + deps.getConvoOwnership(userId, parentConversationId, tenantId ?? null), + deps.getSubagentThreadForParent({ + user: userId, + parentConversationId, + conversationId: threadId, + ...(tenantId == null ? {} : { tenantId }), + }), + ]); + const lineage = child?.subagentThread; + const lease = child?.subagentThreadLease; + const authorized = + parent != null && + child != null && + lineage?.parentConversationId === parentConversationId && + parent.tenantId === tenantId && + child.tenantId === tenantId && + lease?.taskId === taskId && + lease.expiresAt > new Date(); + if (!authorized) { + notFound(res); + return; + } + if (closed || req.destroyed || res.destroyed) return; + + res.setHeader('Content-Encoding', 'identity'); + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + res.flushHeaders?.(); + + heartbeat = setInterval(() => { + if (!res.writableEnded && !res.write(': keep-alive\n\n')) { + close(); + res.end(); + } + }, HEARTBEAT_MS); + heartbeat.unref?.(); + try { + subscription = stream.subscribe(threadId, taskId, { + onEvent: (event) => { + if (!writeSse(res, event)) { + close(); + res.end(); + } + }, + onDone: (event) => { + close(); + writeSse(res, event); + res.end(); + }, + onError: () => { + close(); + writeSse(res, { error: 'Subagent activity stream unavailable' }); + res.end(); + }, + }); + await subscription.ready; + } catch (error) { + /** Release the failed attachment while leaving `closed` to represent only a + * client/response close; the outer catch still owns the SSE error and end. */ + dispose(); + throw error; + } + if (closed || res.destroyed) return; + const durableTerminal = await terminalTaskStatus(deps, userId, threadId, taskId, tenantId); + if (closed || res.destroyed) return; + if (durableTerminal != null) { + close(); + writeSse(res, { + final: true, + subagentActivity: true, + status: durableTerminal, + }); + res.end(); + return; + } + if (!writeSse(res, { ready: true })) { + close(); + res.end(); + } + } catch (error) { + if (closed || res.destroyed) return; + logger.error('[subagentActivity] Failed to open child activity stream', error); + if (!res.headersSent) { + res.status(500).json({ error: 'Failed to open subagent activity stream' }); + return; + } + writeSse(res, { error: 'Subagent activity stream unavailable' }); + res.end(); + } + }; +} + +export const SUBAGENT_ACTIVITY_STREAM_LIMITS: Readonly<{ + eventBytes: number; + labelBytes: number; +}> = Object.freeze({ + eventBytes: MAX_EVENT_BYTES, + labelBytes: MAX_LABEL_BYTES, +}); diff --git a/packages/api/src/agents/subagentCrossReplica.integration.spec.ts b/packages/api/src/agents/subagentCrossReplica.integration.spec.ts index 89e970fbfe..64703793d1 100644 --- a/packages/api/src/agents/subagentCrossReplica.integration.spec.ts +++ b/packages/api/src/agents/subagentCrossReplica.integration.spec.ts @@ -24,6 +24,8 @@ import { import { buildSubagentThreadTaskConfig, SubagentThreadTaskStore } from './subagentThreads'; import { __resetShutdownStateForTests } from '../app/shutdown'; import { createAgentTriggerService } from './triggers/service'; +import { SubagentActivityStream } from './subagentActivity'; +import { RedisEventTransport } from '~/stream'; const DB_SETUP_TIMEOUT_MS = 60_000; const REDIS_URI = process.env.REDIS_URI; @@ -86,6 +88,14 @@ async function createRoutingTransport(instanceId: string, namespace: string) { }); } +async function createActivityStream(): Promise { + const [publisher, subscriber] = await Promise.all([ + connectedRedisClient(), + connectedRedisClient(), + ]); + return new SubagentActivityStream(new RedisEventTransport(publisher, subscriber)); +} + function taskRequest( scopeId: string, input: string, @@ -144,7 +154,10 @@ afterEach(async () => { triggerService = undefined; __resetShutdownStateForTests(); await Promise.all( - taskStores.splice(0).map((store) => store.destroyTaskControlTransport().catch(() => undefined)), + taskStores.splice(0).map(async (store) => { + await store.destroyTaskControlTransport().catch(() => undefined); + store.destroyActivityStream(); + }), ); await Promise.all(redisClients.splice(0).map((client) => client.quit().catch(() => undefined))); await mongoose.connection.db?.dropDatabase(); @@ -264,6 +277,8 @@ describeWithRedis('subagent cross-replica orchestration', () => { await requesterStore.configureTaskControlTransport( await createRoutingTransport('delivery-owner', namespace), ); + ownerStore.configureActivityStream(await createActivityStream()); + requesterStore.configureActivityStream(await createActivityStream()); const config = buildSubagentThreadTaskConfig(ownerStore, { userId, tenantId, @@ -275,10 +290,26 @@ describeWithRedis('subagent cross-replica orchestration', () => { const childRun = (result: string) => { let markEntered = (): void => undefined; entered.push(new Promise((resolve) => (markEntered = resolve))); - return async (_runtime: SubagentTaskRuntime) => { + return async (runtime: SubagentTaskRuntime) => { markEntered(); return new Promise<{ content: string }>((resolve) => - releases.push(() => resolve({ content: result })), + releases.push(() => { + runtime.reportProgress({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: runtime.taskId, + subagentType: 'worker', + subagentKind: 'agent', + subagentAgentId: 'agent-worker', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: result }] } }, + timestamp: new Date().toISOString(), + }); + resolve({ content: result }); + }), ); }; }; @@ -302,6 +333,20 @@ describeWithRedis('subagent cross-replica orchestration', () => { const firstTaskId = accepted(first).task.taskId; const secondTaskId = accepted(second).task.taskId; + const remoteActivity: unknown[] = []; + let resolveRemoteDone!: (status: string) => void; + const remoteDone = new Promise((resolve) => { + resolveRemoteDone = resolve; + }); + const remoteSubscription = requesterStore.subscribeActivity( + accepted(first).task.threadId!, + firstTaskId, + { + onEvent: (event) => remoteActivity.push(event), + onDone: (event) => resolveRemoteDone(event.status), + }, + ); + await remoteSubscription.ready; await expect(requesterStore.listTasks(config.scopeId)).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ taskId: firstTaskId, status: 'running' }), @@ -318,6 +363,13 @@ describeWithRedis('subagent cross-replica orchestration', () => { ).resolves.toMatchObject({ status: 'accepted' }); releases.forEach((release) => release()); + await expect(remoteDone).resolves.toBe('completed'); + expect(remoteActivity).toEqual([ + expect.objectContaining({ + event: 'on_subagent_update', + data: expect.objectContaining({ subagentRunId: firstTaskId }), + }), + ]); await waitUntil(() => { const tasks = [ ownerStore.get(config.scopeId, firstTaskId), diff --git a/packages/api/src/agents/subagentThreads.spec.ts b/packages/api/src/agents/subagentThreads.spec.ts index 9647af676a..b8ca282635 100644 --- a/packages/api/src/agents/subagentThreads.spec.ts +++ b/packages/api/src/agents/subagentThreads.spec.ts @@ -19,6 +19,7 @@ import type { SubagentTaskSnapshot, SubagentTaskStartRequest, SubagentTaskStartResult, + SubagentUpdateEvent, } from '@librechat/agents'; import type { AllMethods, IConversation, IMessage } from '@librechat/data-schemas'; import type { BaseMessage } from '@librechat/agents/langchain/messages'; @@ -27,6 +28,7 @@ import type { SubagentTaskControlTransport, } from './subagentTaskRouting'; import type { SubagentTaskWakeupRegistration } from './subagentThreads'; +import type { IEventTransport } from '~/stream/interfaces/IJobStore'; import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; import { buildSubagentThreadTaskConfig, @@ -36,6 +38,7 @@ import { import { SubagentTaskOwnerUnavailableError } from './subagentTaskRouting'; import { SUBAGENT_COMPLETION_DELIVERY } from './subagentDelivery'; import { createSubagentAttemptKey } from './subagentThreadIds'; +import { SubagentActivityStream } from './subagentActivity'; import { createSubagentUsageSink } from './usage'; let mongod: MongoMemoryServer; @@ -346,6 +349,269 @@ describe('SubagentThreadTaskStore', () => { }); }); + it('streams child activity and closes only after the terminal result is durable', async () => { + const userId = 'activity-stream-user'; + const parentConversationId = randomUUID(); + await saveParent(userId, parentConversationId); + const store = new SubagentThreadTaskStore(methods); + const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + const defaultRun = taskRequest(config.scopeId).run; + const progress: SubagentUpdateEvent = { + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [ + { + subagentRunId: 'parent-run', + subagentType: 'parent', + subagentKind: 'agent', + subagentAgentId: 'parent-agent', + parentRunId: 'root-run', + }, + ], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'Working.' }] } }, + timestamp: '2026-08-21T20:00:00.000Z', + }; + const run = jest.fn(async (...args: Parameters) => { + args[0].reportProgress(progress); + return defaultRun(...args); + }); + + const started = store.start(taskRequest(config.scopeId, { run })); + const accepted = requireAccepted(started); + const events: unknown[] = []; + let resolveTerminal!: (status: string) => void; + const terminal = new Promise((resolve) => { + resolveTerminal = resolve; + }); + store.subscribeActivity(requireThreadId(started), accepted.task.taskId, { + onEvent: (event) => events.push(event), + onDone: (event) => resolveTerminal(event.status), + }); + + await expect(terminal).resolves.toBe('completed'); + expect(events).toEqual([ + { + event: 'on_subagent_update', + data: expect.objectContaining({ + activityEventId: `${accepted.task.taskId}:0`, + activitySequence: 0, + data: progress.data, + }), + }, + ]); + const messages = await methods.getMessages( + { + user: userId, + conversationId: requireThreadId(started), + messageId: `${accepted.task.taskId}:assistant`, + }, + '+subagentTask', + ); + expect(messages).toHaveLength(1); + expect(messages[0]?.subagentTask?.status).toBe('completed'); + }); + + it('drains admitted activity before publishing the terminal event', async () => { + const userId = 'activity-stream-drain-user'; + const parentConversationId = randomUUID(); + await saveParent(userId, parentConversationId); + const store = new SubagentThreadTaskStore(methods); + const publicationOrder: string[] = []; + let releaseFirst!: () => void; + const firstPublication = new Promise((resolve) => (releaseFirst = resolve)); + let publicationCount = 0; + const emitChunk = jest.fn((_streamId: string, event: unknown): Promise => { + publicationCount += 1; + const label = (event as { data?: { label?: string } }).data?.label ?? 'unknown'; + publicationOrder.push(label); + return publicationCount === 1 ? firstPublication : Promise.resolve(); + }); + const emitDone = jest.fn(async () => { + publicationOrder.push('done'); + }); + const transport = { + emitChunk, + emitDone, + emitError: async () => undefined, + subscribe: () => ({ unsubscribe: () => undefined }), + getSubscriberCount: () => 0, + isFirstSubscriber: () => true, + onAllSubscribersLeft: () => undefined, + cleanup: () => undefined, + getTrackedStreamIds: () => [], + destroy: () => undefined, + } satisfies IEventTransport; + store.configureActivityStream(new SubagentActivityStream(transport)); + const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + const defaultRun = taskRequest(config.scopeId).run; + const run = jest.fn(async (...args: Parameters) => { + for (const label of ['first', 'second']) { + args[0].reportProgress({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + depth: 1, + ancestry: [], + phase: 'message_delta', + label, + timestamp: '2026-08-21T20:00:00.000Z', + }); + } + return defaultRun(...args); + }); + + const started = store.start(taskRequest(config.scopeId, { run })); + await waitUntil(() => emitChunk.mock.calls.length === 1, 'first activity publication'); + await waitForSettled(store, config.scopeId, started); + expect(emitChunk).toHaveBeenCalledTimes(1); + + releaseFirst(); + await waitUntil(() => emitDone.mock.calls.length === 1, 'terminal activity delivery'); + + expect(emitChunk).toHaveBeenCalledTimes(2); + expect(publicationOrder).toEqual(['first', 'second', 'done']); + }); + + it('keeps activity delivery observational when its transport is unavailable', async () => { + const userId = 'activity-stream-failure-user'; + const parentConversationId = randomUUID(); + await saveParent(userId, parentConversationId); + const store = new SubagentThreadTaskStore(methods); + const emitChunk = jest.fn(() => { + throw new Error('activity transport unavailable'); + }); + const unavailableTransport = { + emitChunk, + emitDone: async () => Promise.reject(new Error('activity transport unavailable')), + emitError: async () => undefined, + subscribe: () => ({ unsubscribe: () => undefined }), + getSubscriberCount: () => 0, + isFirstSubscriber: () => true, + onAllSubscribersLeft: () => undefined, + cleanup: () => undefined, + getTrackedStreamIds: () => [], + destroy: () => undefined, + } satisfies IEventTransport; + store.configureActivityStream(new SubagentActivityStream(unavailableTransport)); + const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + const defaultRun = taskRequest(config.scopeId).run; + const run = jest.fn(async (...args: Parameters) => { + args[0].reportProgress({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [], + phase: 'start', + timestamp: '2026-08-21T20:00:00.000Z', + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + args[0].reportProgress({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + parentToolCallId: 'tool-call', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'after-error' }] } }, + timestamp: '2026-08-21T20:00:00.010Z', + }); + return defaultRun(...args); + }); + + const started = store.start(taskRequest(config.scopeId, { run })); + await waitForSettled(store, config.scopeId, started); + + expect(store.get(config.scopeId, requireAccepted(started).task.taskId)?.status).toBe( + 'completed', + ); + expect(emitChunk).toHaveBeenCalledTimes(1); + }); + + it('bounds stalled activity and still attempts terminal delivery', async () => { + const userId = 'activity-stream-stalled-user'; + const parentConversationId = randomUUID(); + await saveParent(userId, parentConversationId); + const store = new SubagentThreadTaskStore(methods); + const never = () => new Promise(() => undefined); + const emitChunk = jest.fn(never); + const emitDone = jest.fn(async () => undefined); + const stalledTransport = { + emitChunk, + emitDone, + emitError: async () => undefined, + subscribe: () => ({ unsubscribe: () => undefined }), + getSubscriberCount: () => 0, + isFirstSubscriber: () => true, + onAllSubscribersLeft: () => undefined, + cleanup: () => undefined, + getTrackedStreamIds: () => [], + destroy: () => undefined, + } satisfies IEventTransport; + store.configureActivityStream(new SubagentActivityStream(stalledTransport)); + const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + const defaultRun = taskRequest(config.scopeId).run; + const run = jest.fn(async (...args: Parameters) => { + for (let index = 0; index < 100; index += 1) { + args[0].reportProgress({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: `chunk-${index}` }] } }, + timestamp: '2026-08-21T20:00:00.000Z', + }); + } + await new Promise((resolve) => setTimeout(resolve, 1_100)); + args[0].reportProgress({ + runId: 'root-run', + parentRunId: 'parent-run', + subagentRunId: 'child-run', + subagentType: 'researcher-agent', + subagentKind: 'agent', + subagentAgentId: 'agent-1', + depth: 1, + ancestry: [], + phase: 'message_delta', + data: { delta: { content: [{ type: 'text', text: 'after-timeout' }] } }, + timestamp: '2026-08-21T20:00:01.100Z', + }); + return defaultRun(...args); + }); + + const started = store.start(taskRequest(config.scopeId, { run })); + await waitForSettled(store, config.scopeId, started); + + expect(store.get(config.scopeId, requireAccepted(started).task.taskId)?.status).toBe( + 'completed', + ); + await waitUntil(() => emitDone.mock.calls.length === 1, 'terminal activity delivery'); + expect(emitChunk).toHaveBeenCalledTimes(1); + }); + it('fails before provider work and keeps the durable failure collectable when registration fails', async () => { const userId = 'wakeup-failure-user'; const parentConversationId = randomUUID(); @@ -1480,29 +1746,36 @@ describe('SubagentThreadTaskStore', () => { const userId = 'timeout-user'; const parentConversationId = randomUUID(); await saveParent(userId, parentConversationId); - const store = new SubagentThreadTaskStore(methods, { taskTimeoutMs: 20 }); + const taskTimeoutMs = 60_000; + const timeoutSpy = jest.spyOn(global, 'setTimeout'); + const store = new SubagentThreadTaskStore(methods, { taskTimeoutMs }); const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId }); + let markEntered = (): void => undefined; + const entered = new Promise((resolve) => { + markEntered = resolve; + }); const started = store.start( taskRequest(config.scopeId, { input: 'Run until timeout.', run: async (runtime) => new Promise((_resolve, reject) => { + markEntered(); runtime.signal.addEventListener('abort', () => reject(runtime.signal.reason), { once: true, }); }), }), ); + await entered; + const taskTimeout = timeoutSpy.mock.calls.find((call) => call[1] === taskTimeoutMs)?.[0]; + timeoutSpy.mockRestore(); + expect(taskTimeout).toBeDefined(); + (taskTimeout as () => void)(); await waitForSettled(store, config.scopeId, started); - for (let attempt = 0; attempt < 200; attempt += 1) { - if (!store.isThreadActiveForOwner(userId, requireThreadId(started))) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - if (attempt === 199) { - throw new Error('Timed-out child execution did not finish durable settlement.'); - } - } + await waitUntil( + () => !store.isThreadActiveForOwner(userId, requireThreadId(started)), + 'timed-out child durable settlement', + ); expect(store.claim(config.scopeId, requireAccepted(started).task.taskId)).toMatchObject({ status: 'error', diff --git a/packages/api/src/agents/subagentThreads.ts b/packages/api/src/agents/subagentThreads.ts index a507f04d2c..08053c0c49 100644 --- a/packages/api/src/agents/subagentThreads.ts +++ b/packages/api/src/agents/subagentThreads.ts @@ -15,6 +15,7 @@ import type { SubagentTaskSnapshot, SubagentTaskStartRequest, SubagentTaskStartResult, + SubagentUpdateEvent, } from '@librechat/agents'; import type { AllMethods, @@ -26,6 +27,12 @@ import type { SubagentTaskResultClaim, } from '@librechat/data-schemas'; import type { BaseMessage, StoredMessage } from '@librechat/agents/langchain/messages'; +import type { + SubagentActivityUpdateEvent, + SubagentActivitySubscriber, + SubagentActivitySubscription, + SubagentActivityTerminalStatus, +} from './subagentActivity'; import type { SubagentTaskControlTransport } from './subagentTaskRouting'; import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; import type { HostSubagentTaskConfig } from './subagentDelivery'; @@ -35,10 +42,12 @@ import { controlFingerprint, SubagentTaskOwnerUnavailableError, } from './subagentTaskRouting'; +import { boundSubagentActivityUpdate, SubagentActivityStream } from './subagentActivity'; import { createSubagentAttemptKey, createSubagentThreadId } from './subagentThreadIds'; import { runWithDetachedSubagentUsage } from './subagentTaskContext'; import { SUBAGENT_COMPLETION_DELIVERY } from './subagentDelivery'; import { createConcurrencyLimiter } from '~/utils/promise'; +import { InMemoryEventTransport } from '~/stream'; import { aggregateEmittedUsage } from './usage'; const SCOPE_VERSION = 1; @@ -53,6 +62,10 @@ const DEFAULT_OWNER_DRAIN_POLL_MS = 100; const DELETION_CANCEL_CONCURRENCY = 32; /** Bounds retained control invocations; one entry per applied command. */ const MAX_CONTROL_INVOCATIONS = 4_096; +/** Bounds retained live-only updates while an event transport is unavailable. */ +const MAX_PENDING_ACTIVITY_EVENTS = 32; +/** Live activity must never delay terminal notification indefinitely. */ +const ACTIVITY_PUBLICATION_TIMEOUT_MS = 1_000; /** A cancellation target set resolved before the conversations are removed. */ export interface SubagentCancellationPlan { @@ -124,6 +137,13 @@ interface TaskThreadLease { taskId: string; running: boolean; settling: boolean; + /** Ordered observational tail; canonical child settlement never awaits it. */ + activityTail?: Promise; + activityPending?: number; + /** Terminal settlement stops new admission but must not discard admitted events. */ + activityAdmissionClosed?: boolean; + /** A failed observational publication suppresses the remainder of this task's queue. */ + activityCircuitOpen?: boolean; shared?: { token: string; lost: boolean; @@ -134,6 +154,31 @@ interface TaskThreadLease { }; } +class SubagentActivityPublicationTimeoutError extends Error { + constructor() { + super('Subagent activity publication timed out.'); + this.name = 'SubagentActivityPublicationTimeoutError'; + } +} + +async function settleActivityWithin(operation: Promise): Promise { + let timeout: ReturnType | undefined; + try { + await Promise.race([ + operation, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new SubagentActivityPublicationTimeoutError()), + ACTIVITY_PUBLICATION_TIMEOUT_MS, + ); + timeout.unref?.(); + }), + ]); + } finally { + if (timeout != null) clearTimeout(timeout); + } +} + export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStoreOptions { maxThreadDepth?: number; leaseTtlMs?: number; @@ -430,6 +475,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { private readonly releaseOwnerAdmission?: (userId: string, token: string) => Promise; private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared']; private taskControlTransport?: SubagentTaskControlTransport; + private activityStream = new SubagentActivityStream(new InMemoryEventTransport()); constructor( private readonly methods: SubagentThreadMethods, @@ -484,6 +530,81 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { await transport?.destroy(); } + /** Replaces the process-local activity bus after the host's Redis service is ready. */ + configureActivityStream(stream: SubagentActivityStream): void { + const previous = this.activityStream; + this.activityStream = stream; + previous.destroy(); + } + + destroyActivityStream(): void { + this.activityStream.destroy(); + } + + prepareActivityForShutdown(): void { + this.activityStream.prepareForShutdown(); + } + + subscribeActivity( + threadId: string, + taskId: string, + subscriber: SubagentActivitySubscriber, + ): SubagentActivitySubscription { + return this.activityStream.subscribe(threadId, taskId, subscriber); + } + + private publishActivity( + lease: TaskThreadLease, + threadId: string, + taskId: string, + event: SubagentUpdateEvent, + ): void { + if ( + lease.activityAdmissionClosed === true || + lease.activityCircuitOpen === true || + (lease.activityPending ?? 0) >= MAX_PENDING_ACTIVITY_EVENTS + ) { + return; + } + lease.activityPending = (lease.activityPending ?? 0) + 1; + const boundedEvent = boundSubagentActivityUpdate(event); + const publication = (lease.activityTail ?? Promise.resolve()) + .then(() => { + if (lease.activityCircuitOpen === true) return; + return settleActivityWithin(this.activityStream.publish(threadId, taskId, boundedEvent)); + }) + .catch((error) => { + /** Any failed observational command opens the per-task circuit. Retrying every + * token during an outage only creates command/log pressure; durable state remains. */ + lease.activityCircuitOpen = true; + logger.warn('[subagentThreads] Failed to publish child activity', error); + }) + .finally(() => { + lease.activityPending = Math.max(0, (lease.activityPending ?? 1) - 1); + }); + lease.activityTail = publication; + } + + private completeActivity( + lease: TaskThreadLease, + threadId: string, + taskId: string, + status: SubagentActivityTerminalStatus, + ): void { + lease.activityAdmissionClosed = true; + const terminal = (lease.activityTail ?? Promise.resolve()) + .then(() => settleActivityWithin(this.activityStream.complete(threadId, taskId, status))) + .catch((error) => { + logger.warn('[subagentThreads] Failed to close child activity stream', error); + }); + lease.activityTail = terminal; + void terminal.finally(() => { + if (lease.activityTail === terminal) { + lease.activityTail = undefined; + } + }); + } + /** Gates child creation on the ordinary parent write without retaining request state. */ registerParentPersistence(scopeId: string, persistence: Promise): void { const scope = parseScope(scopeId); @@ -541,6 +662,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { lease.running = true; const detachedUsage: UsageMetadata[] = []; let prepared: PreparedThread | undefined; + let activityTerminal: SubagentActivityTerminalStatus = 'failed'; try { if (runtime.signal.aborted) { throw runtime.signal.reason ?? new Error('Subagent task was cancelled.'); @@ -584,8 +706,27 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { ); } const preparedThread = prepared; + let activitySequence = 0; + const activityRuntime: SubagentTaskRuntime = { + ...runtime, + reportProgress: (event) => { + const sequence = activitySequence++; + const activityEvent: SubagentActivityUpdateEvent = { + ...event, + activityEventId: `${runtime.taskId}:${sequence}`, + activitySequence: sequence, + }; + runtime.reportProgress(activityEvent); + this.publishActivity( + lease, + preparedThread.conversation.conversationId, + runtime.taskId, + activityEvent, + ); + }, + }; const result = await runWithDetachedSubagentUsage(detachedUsage, () => - request.run(runtime, preparedThread.initialMessages), + request.run(activityRuntime, preparedThread.initialMessages), ); if (runtime.signal.aborted) { throw runtime.signal.reason ?? new Error('Subagent task was cancelled.'); @@ -604,6 +745,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { result, detachedUsage, ); + activityTerminal = 'completed'; return result; } catch (error) { /** A replay is already terminal in Mongo. A temporary wakeup-queue @@ -615,6 +757,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { lease.shared == null || (await this.renewSharedLease(scope, threadId, lease)); const terminalTask = this.get(request.scopeId, runtime.taskId); if (runtime.signal.aborted && terminalTask?.status === 'cancelled') { + activityTerminal = 'cancelled'; if (mayPersist) { await this.persistCancellation( scope, @@ -653,6 +796,14 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore { } throw new Error(publicFailureDetail(error)); } finally { + if (prepared != null && prepared.replay == null) { + this.completeActivity( + lease, + prepared.conversation.conversationId, + runtime.taskId, + activityTerminal, + ); + } await this.stopAndReleaseSharedLease(scope, threadId, lease); if (this.activeThreads.get(lockKey) === lease) { this.activeThreads.delete(lockKey); diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts index 4e2e9e5c87..c7fee2ed77 100644 --- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts +++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts @@ -1,9 +1,9 @@ import { logger } from '@librechat/data-schemas'; import type { Redis } from 'ioredis'; +import { emitChunkWithReceipt, emitObservedChunk } from '~/stream/internal/chunkPublication'; import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport'; import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; -import { emitChunkWithReceipt } from '~/stream/internal/chunkPublication'; import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; import { createMockPublisher } from './helpers/publisher'; @@ -261,6 +261,372 @@ describe('RedisEventTransport', () => { transport.destroy(); }); + it('does not let a stale subscription synchronize replacement stream state', async () => { + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'stale-subscription-sync'; + const stale = transport.subscribe( + streamId, + { onChunk: jest.fn() }, + { deferSequenceDelivery: true }, + ); + await stale.ready; + stale.unsubscribe(); + transport.cleanup(streamId); + + const replacement = transport.subscribe( + streamId, + { onChunk: jest.fn() }, + { deferSequenceDelivery: true }, + ); + await replacement.ready; + mockPublisher.get.mockResolvedValue('0'); + + await stale.syncReorderBuffer?.(); + expect(mockPublisher.get).not.toHaveBeenCalled(); + await replacement.syncReorderBuffer?.(); + expect(mockPublisher.get).toHaveBeenCalledTimes(1); + + replacement.unsubscribe(); + transport.destroy(); + }); + + it('fences a fresh channel before advancing past attachment-time chunks', async () => { + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'fresh-attachment-frontier'; + const received: object[] = []; + const messageHandler = getMessageHandler(mockSubscriber); + mockPublisher.get.mockResolvedValueOnce('6'); + mockPublisher.publish.mockImplementation(async (channel: string, payload: string) => { + const parsed = JSON.parse(payload) as { type?: string }; + if (parsed.type === 'subscription_frontier') { + deliverSequencedMessage(messageHandler, streamId, { + type: 'chunk', + seq: 5, + data: { index: 5 }, + }); + messageHandler(channel, payload); + } + return 1; + }); + + const subscription = transport.subscribe( + streamId, + { onChunk: (event) => received.push(event as object) }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + expect(mockSubscriber.subscribe).toHaveBeenCalledTimes(1); + await subscription.ready; + + await subscription.syncReorderBuffer?.(); + + expect(received).toEqual([{ index: 5 }]); + expect(mockPublisher.get).toHaveBeenCalledTimes(1); + subscription.unsubscribe(); + transport.destroy(); + }); + + it('does not capture a frontier after the last subscriber leaves during SUBSCRIBE', async () => { + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + let releaseSubscribe!: () => void; + mockSubscriber.subscribe.mockImplementationOnce( + () => new Promise((resolve) => (releaseSubscribe = resolve)), + ); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + + const subscription = transport.subscribe( + 'closed-frontier-attachment', + { onChunk: jest.fn() }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + subscription.unsubscribe(); + releaseSubscribe(); + await subscription.ready; + + expect(mockPublisher.eval).not.toHaveBeenCalled(); + expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(1); + transport.destroy(); + }); + + it('times out a hung channel subscription and releases deferred delivery', async () => { + jest.useFakeTimers(); + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + let finishOriginalSubscribe!: () => void; + mockSubscriber.subscribe.mockImplementationOnce( + () => new Promise((resolve) => (finishOriginalSubscribe = resolve)), + ); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'hung-channel-attachment'; + const received: object[] = []; + const messageHandler = getMessageHandler(mockSubscriber); + + try { + const subscription = transport.subscribe( + streamId, + { onChunk: (event) => received.push(event as object) }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + jest.advanceTimersByTime(500); + const concurrent = transport.subscribe(streamId, { onChunk: jest.fn() }); + deliverSequencedMessage(messageHandler, streamId, { + type: 'chunk', + seq: 0, + data: { index: 0 }, + }); + expect(received).toEqual([]); + + const readinessFailures = Promise.all([ + subscription.ready?.then( + () => undefined, + (error: unknown) => error, + ), + concurrent.ready?.then( + () => undefined, + (error: unknown) => error, + ), + ]); + jest.advanceTimersByTime(2_500); + const [initiatingFailure, concurrentFailure] = await readinessFailures; + expect(initiatingFailure).toEqual( + expect.objectContaining({ message: expect.stringContaining('Timed out synchronizing') }), + ); + expect(concurrentFailure).toEqual( + expect.objectContaining({ message: expect.stringContaining('Timed out synchronizing') }), + ); + expect(received).toEqual([{ index: 0 }]); + + const retry = transport.subscribe(streamId, { onChunk: jest.fn() }); + await retry.ready; + expect(mockSubscriber.subscribe).toHaveBeenCalledTimes(2); + + finishOriginalSubscribe(); + await Promise.resolve(); + await Promise.resolve(); + expect(mockSubscriber.unsubscribe).not.toHaveBeenCalled(); + + subscription.unsubscribe(); + concurrent.unsubscribe(); + retry.unsubscribe(); + expect(transport.getSubscriberCount(streamId)).toBe(0); + expect(mockSubscriber.unsubscribe).toHaveBeenCalledWith(`stream:{${streamId}}:events`); + } finally { + transport.destroy(); + jest.useRealTimers(); + } + }); + + it('unsubscribes an evicted channel operation that completes without an owner', async () => { + jest.useFakeTimers(); + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + let finishSubscribe!: () => void; + mockSubscriber.subscribe.mockImplementationOnce( + () => new Promise((resolve) => (finishSubscribe = resolve)), + ); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'late-channel-without-owner'; + + try { + const subscription = transport.subscribe( + streamId, + { onChunk: jest.fn() }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + const readinessFailure = subscription.ready?.then( + () => undefined, + (error: unknown) => error, + ); + jest.advanceTimersByTime(3_000); + await readinessFailure; + subscription.unsubscribe(); + expect(transport.getSubscriberCount(streamId)).toBe(0); + expect(mockSubscriber.unsubscribe).not.toHaveBeenCalled(); + + finishSubscribe(); + await Promise.resolve(); + await Promise.resolve(); + + expect(mockSubscriber.unsubscribe).toHaveBeenCalledWith(`stream:{${streamId}}:events`); + } finally { + transport.destroy(); + jest.useRealTimers(); + } + }); + + it('promotes a late active predecessor when its pending replacement fails', async () => { + jest.useFakeTimers(); + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + let finishOriginal!: () => void; + let failReplacement!: (error: Error) => void; + mockSubscriber.subscribe + .mockImplementationOnce(() => new Promise((resolve) => (finishOriginal = resolve))) + .mockImplementationOnce(() => new Promise((_, reject) => (failReplacement = reject))); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'late-active-before-replacement-failure'; + + try { + const original = transport.subscribe( + streamId, + { onChunk: jest.fn() }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + const originalFailure = original.ready?.then( + () => undefined, + (error: unknown) => error, + ); + jest.advanceTimersByTime(3_000); + await originalFailure; + + const replacement = transport.subscribe(streamId, { onChunk: jest.fn() }); + const replacementFailure = replacement.ready?.then( + () => undefined, + (error: unknown) => error, + ); + finishOriginal(); + await Promise.resolve(); + await Promise.resolve(); + failReplacement(new Error('replacement unavailable')); + await replacementFailure; + await Promise.resolve(); + + const survivor = transport.subscribe(streamId, { onChunk: jest.fn() }); + await survivor.ready; + expect(mockSubscriber.subscribe).toHaveBeenCalledTimes(2); + + original.unsubscribe(); + replacement.unsubscribe(); + survivor.unsubscribe(); + expect(mockSubscriber.unsubscribe).toHaveBeenCalledWith(`stream:{${streamId}}:events`); + } finally { + transport.destroy(); + jest.useRealTimers(); + } + }); + + it('releases a surviving subscriber when the initiating frontier capture fails', async () => { + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'failed-frontier-survivor'; + const messageHandler = getMessageHandler(mockSubscriber); + let rejectFrontier!: (error: Error) => void; + mockPublisher.eval.mockImplementationOnce( + () => new Promise((_, reject) => (rejectFrontier = reject)), + ); + + const failed = transport.subscribe( + streamId, + { onChunk: jest.fn() }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + await new Promise((resolve) => setImmediate(resolve)); + + const received: object[] = []; + const survivor = transport.subscribe(streamId, { + onChunk: (event) => received.push(event as object), + }); + await survivor.ready; + deliverSequencedMessage(messageHandler, streamId, { + type: 'chunk', + seq: 0, + data: { index: 0 }, + }); + expect(received).toEqual([]); + + rejectFrontier(new Error('frontier unavailable')); + await expect(failed.ready).rejects.toThrow('frontier unavailable'); + failed.unsubscribe(); + + expect(received).toEqual([{ index: 0 }]); + deliverSequencedMessage(messageHandler, streamId, { + type: 'chunk', + seq: 1, + data: { index: 1 }, + }); + expect(received).toEqual([{ index: 0 }, { index: 1 }]); + + survivor.unsubscribe(); + transport.destroy(); + }); + + it('times out a hung frontier command and releases surviving subscribers', async () => { + jest.useFakeTimers(); + const mockPublisher = createMockPublisher(); + const mockSubscriber = createMockSubscriber(); + const transport = new RedisEventTransport( + mockPublisher as unknown as Redis, + mockSubscriber as unknown as Redis, + ); + const streamId = 'hung-frontier-survivor'; + const messageHandler = getMessageHandler(mockSubscriber); + mockPublisher.eval.mockImplementationOnce((...args: unknown[]) => { + const channel = String(args[3]); + const payload = String(args[4]); + queueMicrotask(() => messageHandler(channel, payload)); + return new Promise(() => undefined); + }); + + try { + const failed = transport.subscribe( + streamId, + { onChunk: jest.fn() }, + { deferSequenceDelivery: true, captureSequenceFrontier: true }, + ); + while (mockPublisher.eval.mock.calls.length === 0) { + await Promise.resolve(); + } + + const received: object[] = []; + const survivor = transport.subscribe(streamId, { + onChunk: (event) => received.push(event as object), + }); + await survivor.ready; + deliverSequencedMessage(messageHandler, streamId, { + type: 'chunk', + seq: 0, + data: { index: 0 }, + }); + expect(received).toEqual([]); + + jest.advanceTimersByTime(3_000); + await expect(failed.ready).rejects.toThrow('Timed out synchronizing Redis subscription'); + expect(received).toEqual([{ index: 0 }]); + + failed.unsubscribe(); + survivor.unsubscribe(); + } finally { + transport.destroy(); + jest.useRealTimers(); + } + }); + it('releases each generation abort subscription after successful completion', async () => { const mockPublisher = createMockPublisher(); const mockSubscriber = createMockSubscriber(); @@ -974,6 +1340,9 @@ describe('RedisEventTransport', () => { await expect( emitChunkWithReceipt(transport, 'failed-stream', { text: 'Hello' }), ).resolves.toBeUndefined(); + await expect( + emitObservedChunk(transport, 'failed-observer', { text: 'Hello' }), + ).rejects.toThrow('Observed chunk publication failed'); transport.destroy(); }); diff --git a/packages/api/src/stream/__tests__/helpers/publisher.ts b/packages/api/src/stream/__tests__/helpers/publisher.ts index de10069e65..9c2c65efab 100644 --- a/packages/api/src/stream/__tests__/helpers/publisher.ts +++ b/packages/api/src/stream/__tests__/helpers/publisher.ts @@ -77,6 +77,11 @@ export function createMockPublisher(): MockPublisher { _allowRetainedEpoch: string, _generationEpochGraceTtl: string, ) => { + if (_numKeys === 1) { + const frontier = await publisher.get(seqKey); + await publisher.publish(jobKey, _generationEpochKey); + return frontier ?? '0'; + } const val = (await publisher.incr(seqKey)) as number; let ttl = Number(ttlSeconds); const seqTtl = (await publisher.ttl(seqKey)) as number; diff --git a/packages/api/src/stream/implementations/InMemoryEventTransport.ts b/packages/api/src/stream/implementations/InMemoryEventTransport.ts index 522e63fed4..5e106f2f8e 100644 --- a/packages/api/src/stream/implementations/InMemoryEventTransport.ts +++ b/packages/api/src/stream/implementations/InMemoryEventTransport.ts @@ -122,6 +122,14 @@ export class InMemoryEventTransport implements IEventTransport { } } + renewDemand(_streamId: string, _ttlMs: number): void { + // The in-process subscriber count is authoritative; no lease is needed. + } + + hasDemand(streamId: string): boolean { + return this.getSubscriberCount(streamId) > 0; + } + async recordProviderDrain( streamId: string, generationId: number, diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.ts index c03d966eef..1de2a227d0 100644 --- a/packages/api/src/stream/implementations/RedisEventTransport.ts +++ b/packages/api/src/stream/implementations/RedisEventTransport.ts @@ -33,6 +33,8 @@ const KEYS = { job: (streamId: string) => `stream:{${streamId}}:job`, /** Latest generation epoch, retained briefly beyond the live job hash. */ generationEpoch: (streamId: string) => `stream:{${streamId}}:generation-epoch`, + /** Short-lived proof that at least one UI is watching this live-only stream. */ + demand: (streamId: string) => `stream:{${streamId}}:demand`, /** Owner-issued proof that this exact generation processed an abort. */ abortAck: (streamId: string, generationId: number) => `stream:{${streamId}}:abort-ack:${generationId}`, @@ -49,6 +51,7 @@ const EventTypes = { CHUNK_BATCH: 'chunk_batch', DONE: 'done', ERROR: 'error', + SUBSCRIPTION_FRONTIER: 'subscription_frontier', ABORT: 'abort', ABORT_ACK: 'abort_ack', PREEMPT: 'preempt', @@ -70,6 +73,8 @@ interface PubSubMessage { abortRequestId?: string; /** Payload for PREEMPT messages; fenced by its own createdAt. */ preempt?: PreemptMessage; + /** Opaque local subscriber fence; never forwarded to application handlers. */ + subscriptionFrontierId?: string; } /** @@ -203,6 +208,13 @@ const PUBLISH_REPLACED_DONE_LUA = 'redis.call("EXPIRE", KEYS[1], ttl) end local seq = val - 1 ' + 'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq) .. ARGV[3]) return seq'; +/** Capture the counter and publish a subscriber-visible fence atomically. Once the + * requesting subscriber observes this marker, every sequenced publication below the + * returned frontier that it could receive is already in its local reorder buffer. */ +const CAPTURE_SUBSCRIPTION_FRONTIER_LUA = + 'local frontier = redis.call("GET", KEYS[1]) or "0" ' + + 'redis.call("PUBLISH", ARGV[1], ARGV[2]) return frontier'; + /** Max messages to buffer before force-flushing (prevents memory issues) */ const MAX_BUFFER_SIZE = 100; /** Rolling-upgrade recovery window after a legacy job hash expires without an epoch marker. */ @@ -210,6 +222,21 @@ const GENERATION_EPOCH_GRACE_TTL_SECONDS = 300; /** Durable owner proof outlives receipt retries and process-local subscriptions. */ const ABORT_ACK_TTL_SECONDS = 86400; const PROVIDER_DRAIN_TTL_SECONDS = 86400; +const SUBSCRIPTION_ATTACHMENT_TIMEOUT_MS = 3_000; + +interface SubscriptionFrontierWaiter { + streamId: string; + resolve: () => void; + reject: (error: Error) => void; + timeout: ReturnType; +} + +interface ChannelSubscriptionState { + ready: Promise; + phase: 'pending' | 'active'; + /** A timed-out predecessor became active while this replacement was pending. */ + fallbackActive: boolean; +} /** * Subscriber state for a stream @@ -265,11 +292,13 @@ export class RedisEventTransport implements IEventTransport { /** Track subscribers per stream */ private streams = new Map(); /** Track channel subscription state: resolved promise = active, pending = in-flight */ - private channelSubscriptions = new Map>(); + private channelSubscriptions = new Map(); /** Counter for generating unique subscriber IDs */ private subscriberIdCounter = 0; /** Coalescable chunk publications awaiting their window flush, per stream */ private pendingBatches = new Map(); + /** Local waiters for atomic sequence-frontier markers published after SUBSCRIBE. */ + private subscriptionFrontierWaiters = new Map(); /** Delta-coalescing window; 0 keeps every publication on the per-event path */ private readonly coalesceWindowMs: number; @@ -299,6 +328,76 @@ export class RedisEventTransport implements IEventTransport { return state; } + private async captureSubscriptionFrontier(streamId: string): Promise { + const subscriptionFrontierId = randomUUID(); + let operationTimeout: ReturnType | undefined; + const observed = new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.subscriptionFrontierWaiters.delete(subscriptionFrontierId); + reject(new Error(`Timed out synchronizing Redis subscription for ${streamId}`)); + }, SUBSCRIPTION_ATTACHMENT_TIMEOUT_MS); + timeout.unref?.(); + this.subscriptionFrontierWaiters.set(subscriptionFrontierId, { + streamId, + resolve, + reject, + timeout, + }); + }); + /** The timeout can win while EVAL is still pending; attach rejection handling now. */ + void observed.catch(() => undefined); + try { + /** Wait for the Redis command and its published marker concurrently. If the command + * commits but its promise never settles, the marker timeout still releases attachment + * admission instead of leaving every surviving local subscriber deferred forever. */ + const operation = Promise.all([ + this.publisher.eval( + CAPTURE_SUBSCRIPTION_FRONTIER_LUA, + 1, + KEYS.sequence(streamId), + CHANNELS.events(streamId), + JSON.stringify({ + type: EventTypes.SUBSCRIPTION_FRONTIER, + subscriptionFrontierId, + }), + ), + observed, + ]); + const operationDeadline = new Promise((_, reject) => { + operationTimeout = setTimeout( + () => reject(new Error(`Timed out synchronizing Redis subscription for ${streamId}`)), + SUBSCRIPTION_ATTACHMENT_TIMEOUT_MS, + ); + operationTimeout.unref?.(); + }); + const [raw] = await Promise.race([operation, operationDeadline]); + const parsed = raw != null ? parseInt(String(raw), 10) : 0; + return Number.isNaN(parsed) ? 0 : parsed; + } catch (error) { + const waiter = this.subscriptionFrontierWaiters.get(subscriptionFrontierId); + if (waiter != null) { + waiter.reject(error instanceof Error ? error : new Error(String(error))); + } + throw error; + } finally { + if (operationTimeout != null) clearTimeout(operationTimeout); + const waiter = this.subscriptionFrontierWaiters.get(subscriptionFrontierId); + if (waiter != null) { + clearTimeout(waiter.timeout); + this.subscriptionFrontierWaiters.delete(subscriptionFrontierId); + } + } + } + + private releaseSubscriptionFrontiers(streamId: string): void { + for (const [id, waiter] of this.subscriptionFrontierWaiters) { + if (waiter.streamId !== streamId) continue; + clearTimeout(waiter.timeout); + this.subscriptionFrontierWaiters.delete(id); + waiter.resolve(); + } + } + /** * Create a new Redis event transport. * @@ -540,25 +639,86 @@ export class RedisEventTransport implements IEventTransport { } } - private ensureChannelSubscription(channel: string): Promise { + private ensureChannelSubscription(streamId: string): Promise { + const channel = CHANNELS.events(streamId); const existing = this.channelSubscriptions.get(channel); if (existing) { - return existing; + return existing.ready; } - const ready = this.subscriber.subscribe(channel).then(() => { + const operation = this.subscriber.subscribe(channel); + let expired = false; + let timeout: ReturnType | undefined; + const state: ChannelSubscriptionState = { + ready: Promise.resolve(), + phase: 'pending', + fallbackActive: false, + }; + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => { + expired = true; + reject(new Error(`Timed out synchronizing Redis subscription for ${channel}`)); + }, SUBSCRIPTION_ATTACHMENT_TIMEOUT_MS); + timeout.unref?.(); + }); + const ready = Promise.race([operation, deadline]).then(() => { + state.phase = 'active'; + state.fallbackActive = false; logger.debug(`[RedisEventTransport] Subscription active for channel ${channel}`); }); - this.channelSubscriptions.set(channel, ready); + state.ready = ready; + operation.then( + () => { + if (timeout != null) clearTimeout(timeout); + if (expired) { + const current = this.channelSubscriptions.get(channel); + if (current == null || current === state) { + if (current === state) this.channelSubscriptions.delete(channel); + this.trackOrReleaseActiveChannel(streamId, channel); + } else if (current.phase === 'pending') { + /** If this replacement later fails, its rejection promotes the known-active + * predecessor instead of losing the only tracked channel subscription. */ + current.fallbackActive = true; + } + } + }, + () => { + if (timeout != null) clearTimeout(timeout); + }, + ); + this.channelSubscriptions.set(channel, state); void ready.catch((err) => { - if (this.channelSubscriptions.get(channel) === ready) { + if (this.channelSubscriptions.get(channel) === state) { this.channelSubscriptions.delete(channel); + if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel); } logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err); }); return ready; } + private trackOrReleaseActiveChannel(streamId: string, channel: string): void { + const streamState = this.streams.get(streamId); + const hasOwner = + streamState != null && + (streamState.count > 0 || + streamState.abortCallbacks.size > 0 || + streamState.abortAckWaiters.size > 0 || + streamState.preemptCallbacks.size > 0); + if (hasOwner) { + /** Re-track the active channel so its surviving owner can release it normally. */ + this.channelSubscriptions.set(channel, { + ready: Promise.resolve(), + phase: 'active', + fallbackActive: false, + }); + return; + } + this.subscriber.unsubscribe(channel).catch((error) => { + logger.error(`[RedisEventTransport] Failed to release late subscription ${channel}:`, error); + }); + } + /** Reset subscriber reorder buffer state to initial values */ private resetReorderBuffer(streamId: string): void { const state = this.streams.get(streamId); @@ -573,6 +733,22 @@ export class RedisEventTransport implements IEventTransport { } } + /** Release a deferred attachment into the ordinary ordered-delivery fallback. + * The state identity fence prevents a failed stale attachment from touching a + * replacement lifecycle that happens to reuse the same stream ID. */ + private releaseDeferredDelivery(streamId: string, expectedState: StreamSubscribers): void { + const state = this.streams.get(streamId); + if (state !== expectedState || !state.reorderBuffer.deliveryDeferred) { + return; + } + const buffer = state.reorderBuffer; + buffer.deliveryDeferred = false; + this.flushPendingMessages(streamId, state); + if (buffer.pending.size > 0) { + this.scheduleFlushTimeout(streamId, state); + } + } + /** * Advance subscriber reorder buffer to the authoritative Redis sequence counter * (cross-replica safe). @@ -582,12 +758,23 @@ export class RedisEventTransport implements IEventTransport { * above it are live chunks from the ongoing generation. Using the exact replay frontier * (not the Redis counter) is critical: INCR can advance the counter past a live chunk's * sequence during the GET window. Undefined means no local replay, so currentSeq is trusted. + * @param preserveBufferedBeforeFrontier - A fresh post-SUBSCRIBE fence has no replay log; + * preserve every frame observed before its marker and begin at the earliest buffered seq. */ - async syncReorderBuffer(streamId: string, replayedNextSeq?: number): Promise { + async syncReorderBuffer( + streamId: string, + replayedNextSeq?: number, + preserveBufferedBeforeFrontier = false, + ): Promise { const initialState = this.streams.get(streamId); try { const key = KEYS.sequence(streamId); - const rawStr = await this.publisher.get(key); + /** The atomic post-SUBSCRIBE marker already returned an authoritative frontier; + * another GET would add a new failure/race point before releasing delivery. */ + const rawStr = + preserveBufferedBeforeFrontier && replayedNextSeq != null + ? String(replayedNextSeq) + : await this.publisher.get(key); const parsed = rawStr != null ? parseInt(rawStr, 10) : 0; const currentSeq = Number.isNaN(parsed) ? 0 : parsed; const state = this.streams.get(streamId); @@ -608,7 +795,7 @@ export class RedisEventTransport implements IEventTransport { // Prune true duplicates already delivered via earlyEventBuffer. Entries at or above // the absolute replay frontier are live (possibly from an ongoing generation). - if (replayedNextSeq != null) { + if (replayedNextSeq != null && !preserveBufferedBeforeFrontier) { for (const seq of buffer.pending.keys()) { if (seq < replayedNextSeq) { buffer.pending.delete(seq); @@ -651,7 +838,7 @@ export class RedisEventTransport implements IEventTransport { const buffer = state.reorderBuffer; // The local replay frontier remains authoritative even when the shared counter // cannot be read. Drop its pub/sub copies before releasing any later live events. - if (replayedNextSeq != null) { + if (replayedNextSeq != null && !preserveBufferedBeforeFrontier) { for (const seq of buffer.pending.keys()) { if (seq < replayedNextSeq) { buffer.pending.delete(seq); @@ -659,11 +846,7 @@ export class RedisEventTransport implements IEventTransport { } buffer.nextSeq = Math.max(buffer.nextSeq, replayedNextSeq); } - buffer.deliveryDeferred = false; - this.flushPendingMessages(streamId, state); - if (buffer.pending.size > 0) { - this.scheduleFlushTimeout(streamId, state); - } + this.releaseDeferredDelivery(streamId, state); } throw err; } @@ -686,6 +869,18 @@ export class RedisEventTransport implements IEventTransport { try { const parsed = JSON.parse(message) as PubSubMessage; + if ( + parsed.type === EventTypes.SUBSCRIPTION_FRONTIER && + parsed.subscriptionFrontierId != null + ) { + const waiter = this.subscriptionFrontierWaiters.get(parsed.subscriptionFrontierId); + if (waiter?.streamId === streamId) { + clearTimeout(waiter.timeout); + this.subscriptionFrontierWaiters.delete(parsed.subscriptionFrontierId); + waiter.resolve(); + } + return; + } /** Aborts, preempts, and abort acknowledgements are consumed by * transport-internal waiters (e.g. pending-ack resolution), not SSE * subscribers, so they must flow even with zero local subscribers. */ @@ -962,6 +1157,7 @@ export class RedisEventTransport implements IEventTransport { private detachStreamSubscribers(streamId: string, state: StreamSubscribers): void { this.resetReorderBuffer(streamId); + this.releaseSubscriptionFrontiers(streamId); this.unsubscribeUnusedChannel(streamId, state); @@ -1008,11 +1204,15 @@ export class RedisEventTransport implements IEventTransport { }, options?: { deferSequenceDelivery?: boolean; + captureSequenceFrontier?: boolean; /** @deprecated Use deferSequenceDelivery. */ deferDeliveryUntilSynchronized?: boolean; }, - ): { unsubscribe: () => void; ready?: Promise } { - const channel = CHANNELS.events(streamId); + ): { + unsubscribe: () => void; + ready?: Promise; + syncReorderBuffer?: () => void | Promise; + } { const subscriberId = `sub_${++this.subscriberIdCounter}`; // Initialize stream state if needed @@ -1028,10 +1228,45 @@ export class RedisEventTransport implements IEventTransport { streamState.count++; streamState.handlers.set(subscriberId, handlers); - const readyPromise = this.ensureChannelSubscription(channel); + /** A fresh activity attachment has no replay log. SUBSCRIBE first, then atomically + * capture the sequence and publish a marker. Observing that marker proves every + * receivable pre-frontier frame is already buffered locally. */ + const captureSequenceFrontier = + options?.captureSequenceFrontier === true && streamState.reorderBuffer.deliveryDeferred; + const channelReady = this.ensureChannelSubscription(streamId); + const attachmentFrontier = captureSequenceFrontier + ? channelReady + .then(() => { + if (this.streams.get(streamId) !== streamState || streamState.count === 0) return; + return this.captureSubscriptionFrontier(streamId); + }) + .catch((error) => { + /** The initiating route may leave while another local viewer remains. + * A failed fence must not strand that survivor behind shared deferral. */ + this.releaseDeferredDelivery(streamId, streamState); + throw error; + }) + : undefined; + const readyPromise = attachmentFrontier?.then(() => undefined) ?? channelReady; return { ready: readyPromise, + syncReorderBuffer: async () => { + /** A delayed attachment must never synchronize state recreated under the same ID. */ + if (this.streams.get(streamId) !== streamState) return; + if (captureSequenceFrontier) { + const capturedFrontier = await attachmentFrontier; + if ( + capturedFrontier == null || + this.streams.get(streamId) !== streamState || + streamState.count === 0 + ) { + return; + } + return this.syncReorderBuffer(streamId, capturedFrontier, true); + } + return this.syncReorderBuffer(streamId); + }, unsubscribe: () => { // An unsubscribe closure belongs to the exact state and handler created // above. After cleanup + stream reuse, it must not decrement or detach @@ -1108,6 +1343,14 @@ export class RedisEventTransport implements IEventTransport { } } + async renewDemand(streamId: string, ttlMs: number): Promise { + await this.publisher.set(KEYS.demand(streamId), '1', 'PX', ttlMs); + } + + async hasDemand(streamId: string): Promise { + return (await this.publisher.exists(KEYS.demand(streamId))) > 0; + } + async emitReplacedDoneConfirmed( streamId: string, event: unknown, @@ -1342,9 +1585,8 @@ export class RedisEventTransport implements IEventTransport { logger.error(`[RedisEventTransport] Failed to inspect generation abort proof:`, error); } - const channel = CHANNELS.events(streamId); const state = this.getOrCreateStreamState(streamId); - await this.ensureChannelSubscription(channel); + await this.ensureChannelSubscription(streamId); if (this.streams.get(streamId) !== state) { return false; } @@ -1384,14 +1626,13 @@ export class RedisEventTransport implements IEventTransport { streamId: string, callback: (generationId?: number) => void | boolean, ): Promise<() => void> { - const channel = CHANNELS.events(streamId); const state = this.getOrCreateStreamState(streamId); const registration = { callback }; state.abortCallbacks.add(registration); try { - await this.ensureChannelSubscription(channel); + await this.ensureChannelSubscription(streamId); } catch (error) { state.abortCallbacks.delete(registration); this.unsubscribeUnusedChannel(streamId, state); @@ -1438,14 +1679,13 @@ export class RedisEventTransport implements IEventTransport { * replacement. */ async onPreempt(streamId: string, callback: (msg: PreemptMessage) => void): Promise<() => void> { - const channel = CHANNELS.events(streamId); const state = this.getOrCreateStreamState(streamId); const registration = { callback }; state.preemptCallbacks.add(registration); try { - await this.ensureChannelSubscription(channel); + await this.ensureChannelSubscription(streamId); } catch (error) { state.preemptCallbacks.delete(registration); this.unsubscribeUnusedChannel(streamId, state); @@ -1480,6 +1720,7 @@ export class RedisEventTransport implements IEventTransport { cleanup(streamId: string): void { const channel = CHANNELS.events(streamId); const state = this.streams.get(streamId); + this.releaseSubscriptionFrontiers(streamId); /** Terminal publications flushed ahead of themselves; anything still pending * here belongs to a torn-down generation and stays recoverable from the @@ -1514,6 +1755,11 @@ export class RedisEventTransport implements IEventTransport { * Destroy all resources. */ destroy(): void { + for (const streamId of new Set( + [...this.subscriptionFrontierWaiters.values()].map((waiter) => waiter.streamId), + )) { + this.releaseSubscriptionFrontiers(streamId); + } for (const streamId of this.pendingBatches.keys()) { this.discardCoalescedChunks(streamId); } diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 33aa8ddd85..771c15d931 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -1357,8 +1357,16 @@ export interface IEventTransport { options?: { /** Hold sequenced events until syncReorderBuffer establishes the replay frontier. */ deferSequenceDelivery?: boolean; + /** After opening a fresh Pub/Sub channel, atomically capture its sequence frontier + * and fence delivery so synchronization cannot lose an attachment-time frame. */ + captureSequenceFrontier?: boolean; }, - ): { unsubscribe: () => void; ready?: Promise }; + ): { + unsubscribe: () => void; + ready?: Promise; + /** Synchronize only the transport state captured by this concrete subscription. */ + syncReorderBuffer?: () => void | Promise; + }; /** * Publish a chunk event. @@ -1379,6 +1387,12 @@ export interface IEventTransport { */ emitError(streamId: string, error: string, generationId?: number): void | Promise; + /** Optional live-view demand marker used by observational streams that do not replay. */ + renewDemand?(streamId: string, ttlMs: number): void | Promise; + + /** Returns whether at least one live viewer recently renewed demand for this stream. */ + hasDemand?(streamId: string): boolean | Promise; + /** * Publish an abort signal to all replicas (Redis mode). * Enables cross-replica abort: user aborts on Replica B, diff --git a/packages/api/src/stream/internal/chunkPublication.ts b/packages/api/src/stream/internal/chunkPublication.ts index 2f97256d9e..d3bc3d5d28 100644 --- a/packages/api/src/stream/internal/chunkPublication.ts +++ b/packages/api/src/stream/internal/chunkPublication.ts @@ -60,3 +60,26 @@ export function emitChunkWithReceipt( } return Promise.resolve(transport.emitChunk(streamId, event, generationId)); } + +/** + * Publish an observational chunk while surfacing operational transport failure. + * + * The public Redis `emitChunk` contract intentionally preserves legacy best-effort + * behavior for generation streaming. Detached activity needs the stronger signal so + * its per-task circuit can stop issuing a failing Redis command for every token. + */ +export async function emitObservedChunk( + transport: IEventTransport, + streamId: string, + event: unknown, +): Promise { + const capability = chunkPublicationCapabilities.get(transport); + if (!capability) { + await transport.emitChunk(streamId, event); + return; + } + const receipt = await capability(streamId, event); + if (receipt === undefined) { + throw new Error('Observed chunk publication failed'); + } +} diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.ts index 24ec03d119..69c7b18c8b 100644 --- a/packages/data-provider/src/types/runs.ts +++ b/packages/data-provider/src/types/runs.ts @@ -378,6 +378,10 @@ export interface SubagentUpdateEvent { runId: string; parentRunId?: string; subagentRunId: string; + /** Host-assigned identity preserved when one detached update overlaps delivery streams. */ + activityEventId?: string; + /** Host-assigned monotonic sequence within one detached child run. */ + activitySequence?: number; /** Parent-side `tool_call_id` for the `subagent` tool invocation that * triggered this run. Surfaces from the SDK (`3.1.67-dev.2`+) so hosts * can correlate child progress to the parent tool call deterministically. */