From e1fc59c9cfbb2bd1fc822a25ad0b874022a794b6 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:04:57 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=A6=E2=80=8D=F0=9F=94=A5=20fix:=20Reco?= =?UTF-8?q?ver=20Redis=20Clients=20After=20Extended=20Outages=20(#18164)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐦‍🔥 fix: Recover Redis Clients After Extended Outages * fix: Retain Subscriber Registrations Across Repeated Disconnects * test: Diagnose Redis Fixture Startup Failures * fix: Close Redis Invalidation Connections During Outages * fix: Bound Redis Subscriber Lifetimes During Shutdown * fix: Retain Pending Redis Subscription Cancellation After Removal --------- Co-authored-by: Lia --- .../agents/subagentTaskRouting.redis.spec.ts | 218 +++++++++++++++++ .../src/agents/subagentTaskRouting.spec.ts | 138 +++++++++++ .../api/src/agents/subagentTaskRouting.ts | 89 +++++-- .../api/src/app/invalidation.redis.spec.ts | 59 ++++- packages/api/src/app/invalidation.spec.ts | 69 ++++++ packages/api/src/app/invalidation.ts | 20 +- .../outage.cache_integration.spec.ts | 217 +++++++++++++++++ .../api/src/cache/__tests__/outage.helper.ts | 220 ++++++++++++++++++ .../api/src/cache/__tests__/outage.spec.ts | 97 ++++++++ packages/api/src/cache/cacheConfig.spec.ts | 33 +++ packages/api/src/cache/cacheConfig.ts | 2 +- packages/api/src/cache/redisClients.ts | 95 +++----- packages/api/src/cache/redisUtils.spec.ts | 37 ++- packages/api/src/cache/redisUtils.ts | 21 +- packages/api/src/cache/retry.spec.ts | 66 ++++++ packages/api/src/cache/retry.ts | 36 +++ .../__tests__/RedisEventTransport.spec.ts | 207 +++++++++++++++- ...sEventTransport.stream_integration.spec.ts | 12 +- ...-reorder-desync.stream_integration.spec.ts | 5 +- .../implementations/RedisEventTransport.ts | 58 +++-- 20 files changed, 1586 insertions(+), 113 deletions(-) create mode 100644 packages/api/src/agents/subagentTaskRouting.redis.spec.ts create mode 100644 packages/api/src/cache/__tests__/outage.cache_integration.spec.ts create mode 100644 packages/api/src/cache/__tests__/outage.helper.ts create mode 100644 packages/api/src/cache/__tests__/outage.spec.ts create mode 100644 packages/api/src/cache/cacheConfig.spec.ts create mode 100644 packages/api/src/cache/retry.spec.ts create mode 100644 packages/api/src/cache/retry.ts diff --git a/packages/api/src/agents/subagentTaskRouting.redis.spec.ts b/packages/api/src/agents/subagentTaskRouting.redis.spec.ts new file mode 100644 index 0000000000..3c4fb6270c --- /dev/null +++ b/packages/api/src/agents/subagentTaskRouting.redis.spec.ts @@ -0,0 +1,218 @@ +import Redis from 'ioredis'; +import http from 'node:http'; +import type { SubagentTaskControlHandler } from './subagentTaskRouting'; +import type { RespServer } from '~/cache/__tests__/resp.helper'; +import { + RedisSubagentTaskControlTransport, + SubagentTaskOwnerUnavailableError, +} from './subagentTaskRouting'; +import { + setupGracefulShutdown, + registerShutdownTask, + __resetShutdownStateForTests, +} from '~/app/shutdown'; +import { createIoRedisSubscriber, duplicateIoRedisClient } from '~/cache/redisUtils'; +import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper'; + +jest.mock('@librechat/data-schemas', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); +jest.mock('~/utils', () => ({ + ...jest.requireActual('~/utils/math'), + ...jest.requireActual('~/utils/common'), +})); + +const handler: SubagentTaskControlHandler = { + claim: () => ({ status: 'not_found' }), + control: () => ({ status: 'not_found' }), + list: () => [], + retainsTaskOwnership: () => false, + cancelScope: () => 0, +}; +const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGHUP']; + +describe('Redis task-routing lifecycle', () => { + let server: RespServer; + let shared: Redis; + let publisher: Redis; + let subscriber: Redis; + let transport: RedisSubagentTaskControlTransport; + let shutdownExit: jest.SpyInstance | undefined; + let originalListeners: Map; + + beforeEach(async () => { + originalListeners = new Map(signals.map((signal) => [signal, process.listeners(signal)])); + __resetShutdownStateForTests(); + shutdownExit = undefined; + server = await startRespServer(); + shared = new Redis(server.url, { + maxRetriesPerRequest: 0, + autoResendUnfulfilledCommands: false, + retryStrategy: () => 20, + }); + shared.on('error', () => undefined); + await waitFor(() => shared.status === 'ready'); + publisher = duplicateIoRedisClient(shared, { enableOfflineQueue: false }) as Redis; + publisher.on('error', () => undefined); + subscriber = createIoRedisSubscriber(shared, 'task-lifecycle-subscriber') as Redis; + transport = new RedisSubagentTaskControlTransport(publisher, subscriber); + }); + + afterEach(async () => { + try { + await transport.destroy(); + subscriber.disconnect(); + publisher.disconnect(); + shared.disconnect(); + await server.close(); + await new Promise((resolve) => setImmediate(resolve)); + if (shutdownExit) await waitFor(() => shutdownExit!.mock.calls.length === 1, 500); + } finally { + __resetShutdownStateForTests(); + for (const signal of signals) { + for (const listener of process.listeners(signal)) { + if (!originalListeners.get(signal)?.includes(listener)) { + process.removeListener(signal, listener); + } + } + } + shutdownExit?.mockRestore(); + } + }); + + it('cancels a readiness wait when destroyed without closing its borrowed publisher', async () => { + await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready'); + await server.close(); + await waitFor( + () => publisher.status === 'reconnecting' && subscriber.status === 'reconnecting', + ); + const publisherDisconnect = jest.spyOn(publisher, 'disconnect'); + const publisherOnce = jest.spyOn(publisher, 'once'); + const subscriberOnce = jest.spyOn(subscriber, 'once'); + let result: unknown; + void transport.bind(handler).then( + () => { + result = 'bound'; + }, + (error: unknown) => { + result = error; + }, + ); + + const readinessListeners = [publisherOnce, subscriberOnce].map((spy) => + spy.mock.calls.filter(([event]) => event === 'ready' || event === 'end'), + ); + let destroyed = false; + void transport.destroy().then(() => { + destroyed = true; + }); + await waitFor(() => result != null && destroyed, 500); + + expect(result).toBeInstanceOf(SubagentTaskOwnerUnavailableError); + for (const [index, client] of [publisher, subscriber].entries()) { + for (const [event, listener] of readinessListeners[index]) { + expect(client.listeners(event)).not.toContain(listener); + } + } + expect(publisherDisconnect).not.toHaveBeenCalled(); + }); + + it('does not subscribe when readiness resolves just before destruction', async () => { + await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready'); + const subscribe = jest.spyOn(subscriber, 'subscribe'); + const binding = transport.bind(handler).then( + () => 'bound', + (error: unknown) => error, + ); + + await transport.destroy(); + + expect(await binding).toBeInstanceOf(SubagentTaskOwnerUnavailableError); + expect(subscribe).not.toHaveBeenCalled(); + expect(subscriber.listenerCount('message')).toBe(0); + await expect(publisher.ping()).resolves.toBe('PONG'); + }); + + it('cancels an unacknowledged initial subscription on destruction', async () => { + await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready'); + server.silent = true; + let result: unknown; + void transport.bind(handler).then( + () => { + result = 'bound'; + }, + (error: unknown) => { + result = error; + }, + ); + await waitFor(() => server.commands.some((frame) => frame[0] === 'subscribe')); + + let destroyed = false; + void transport.destroy().then(() => { + destroyed = true; + }); + await waitFor(() => result != null && destroyed, 500); + + expect(result).toBeInstanceOf(SubagentTaskOwnerUnavailableError); + expect(subscriber.listenerCount('message')).toBe(0); + }); + + it('stops its heartbeat on intentional disconnect even when the closed socket emits no end', async () => { + await transport.bind(handler); + await server.close(); + await waitFor(() => subscriber.status === 'reconnecting'); + const listeners = subscriber.listenerCount('end'); + expect(listeners).toBeGreaterThan(0); + + subscriber.disconnect(true); + expect(subscriber.listenerCount('end')).toBe(listeners); + subscriber.disconnect(); + + expect(subscriber.listenerCount('end')).toBe(listeners - 1); + }); + + it('allows later shutdown tasks to run while Redis is unreachable', async () => { + await transport.bind(handler); + await server.close(); + await waitFor(() => subscriber.status === 'reconnecting'); + const steps: string[] = []; + const exit = jest.spyOn(process, 'exit').mockImplementation((() => undefined) as never); + shutdownExit = exit; + registerShutdownTask( + 'generation job manager', + () => { + steps.push('generation'); + }, + { priority: 100 }, + ); + registerShutdownTask( + 'subagent task store', + async () => { + await transport.destroy(); + publisher.disconnect(); + steps.push('task-routing'); + }, + { priority: 90 }, + ); + registerShutdownTask( + 'favorites persistence', + () => { + steps.push('favorites'); + }, + { priority: -100 }, + ); + setupGracefulShutdown(http.createServer()); + + const listener = process + .listeners('SIGTERM') + .find( + (entry) => !originalListeners.get('SIGTERM')?.includes(entry), + ) as NodeJS.SignalsListener; + listener('SIGTERM'); + await waitFor(() => exit.mock.calls.length === 1, 500); + + expect(steps).toEqual(['generation', 'task-routing', 'favorites']); + expect(exit).toHaveBeenCalledWith(0); + await transport.destroy(); + }); +}); diff --git a/packages/api/src/agents/subagentTaskRouting.spec.ts b/packages/api/src/agents/subagentTaskRouting.spec.ts index 0c7ef0796f..1387323b62 100644 --- a/packages/api/src/agents/subagentTaskRouting.spec.ts +++ b/packages/api/src/agents/subagentTaskRouting.spec.ts @@ -184,6 +184,144 @@ function taskHandler( } describe('RedisSubagentTaskControlTransport', () => { + it('bounds a missing initial SUBSCRIBE acknowledgement and closes only its subscriber', async () => { + jest.useFakeTimers(); + const bus = new FakeRedisBus(); + const publisher = bus.createClient(); + const subscriber = bus.createClient(); + jest.spyOn(subscriber, 'subscribe').mockImplementation(() => new Promise(() => undefined)); + const transport = new RedisSubagentTaskControlTransport( + asRedis(publisher), + asRedis(subscriber), + ); + try { + const binding = transport.bind(taskHandler()).then( + () => 'bound', + (error: unknown) => error, + ); + await jest.advanceTimersByTimeAsync(10_000); + expect(await binding).toBeInstanceOf(SubagentTaskOwnerUnavailableError); + expect(subscriber.disconnected).toBe(true); + expect(subscriber.listeners.size).toBe(0); + expect(publisher.disconnected).toBe(false); + expect(jest.getTimerCount()).toBe(0); + } finally { + await transport.destroy(); + jest.useRealTimers(); + } + }); + + it('ignores a message callback queued before destruction', async () => { + const bus = new FakeRedisBus(); + const subscriber = bus.createClient(); + const transport = new RedisSubagentTaskControlTransport( + asRedis(bus.createClient()), + asRedis(subscriber), + ); + const control = jest.fn(() => ({ status: 'not_found' as const })); + await transport.bind(taskHandler({ control })); + const listener = [...subscriber.listeners][0]; + const channel = [...subscriber.channels][0]; + await transport.destroy(); + + listener( + channel, + JSON.stringify({ + version: 1, + kind: 'request', + operation: 'control', + requestId: 'late-request', + requesterId: 'requester', + scopeId: 'scope-1', + taskId: 'task-1', + invocationId: 'invocation-1', + expiresAt: Date.now() + 1000, + command: { action: 'queue', message: 'late control' }, + }), + ); + await Promise.resolve(); + + expect(control).not.toHaveBeenCalled(); + }); + + it('does not publish a control reply that settles after destruction', async () => { + const bus = new FakeRedisBus(); + const publisher = bus.createClient(); + const subscriber = bus.createClient(); + const transport = new RedisSubagentTaskControlTransport( + asRedis(publisher), + asRedis(subscriber), + ); + let finish!: (value: SubagentTaskControlResult) => void; + const control = jest.fn( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + await transport.bind(taskHandler({ control })); + const channel = [...subscriber.channels][0]; + bus.publish( + channel, + JSON.stringify({ + version: 1, + kind: 'request', + operation: 'control', + requestId: 'pending-request', + requesterId: 'requester', + scopeId: 'scope-1', + taskId: 'task-1', + invocationId: 'invocation-1', + expiresAt: Date.now() + 1000, + command: { action: 'queue', message: 'pending control' }, + }), + ); + await Promise.resolve(); + expect(control).toHaveBeenCalledTimes(1); + await transport.destroy(); + const publish = jest.spyOn(publisher, 'publish'); + + finish({ status: 'not_found' }); + await Promise.resolve(); + await Promise.resolve(); + + expect(publish).not.toHaveBeenCalled(); + }); + + it('does not issue queued owner refreshes after destruction', async () => { + jest.useFakeTimers(); + const bus = new FakeRedisBus(); + const publisher = bus.createClient(); + const transport = new RedisSubagentTaskControlTransport( + asRedis(publisher), + asRedis(bus.createClient()), + { registrationHeartbeatMs: 1 }, + ); + const tasks = Array.from({ length: 36 }, (_, index) => snapshot({ taskId: `task-${index}` })); + const releases: Array<() => void> = []; + try { + await transport.bind(taskHandler({ list: () => tasks })); + for (const task of tasks) await transport.registerTask('scope-1', task.taskId, 60_000); + bus.registrationHook = () => + new Promise((resolve) => { + releases.push(resolve); + }); + const evalCommand = jest.spyOn(publisher, 'eval'); + await jest.advanceTimersByTimeAsync(1); + expect(releases).toHaveLength(32); + + await transport.destroy(); + for (const release of releases) release(); + await jest.advanceTimersByTimeAsync(1); + + expect(evalCommand).toHaveBeenCalledTimes(32); + expect(jest.getTimerCount()).toBe(0); + } finally { + await transport.destroy(); + jest.useRealTimers(); + } + }); + it('waits for the fail-fast publisher before reporting itself bound', async () => { const bus = new FakeRedisBus(); const publisher = new EventEmitter() as EventEmitter & { status: string }; diff --git a/packages/api/src/agents/subagentTaskRouting.ts b/packages/api/src/agents/subagentTaskRouting.ts index 3078eeaa98..0075dfc22d 100644 --- a/packages/api/src/agents/subagentTaskRouting.ts +++ b/packages/api/src/agents/subagentTaskRouting.ts @@ -355,7 +355,13 @@ function boundedControlResult(result: SubagentTaskControlResult): SubagentTaskCo }; } -async function waitForRedisConnectionReady(client: RedisClient): Promise { +async function waitForRedisConnectionReady( + client: RedisClient, + signal: AbortSignal, +): Promise { + if (signal.aborted) { + throw new SubagentTaskOwnerUnavailableError(); + } if (client.status == null || client.status === 'ready') { return; } @@ -367,6 +373,7 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise { clearTimeout(timeout); client.off('ready', onReady); client.off('end', onEnd); + signal.removeEventListener('abort', onEnd); }; const onReady = () => { cleanup(); @@ -381,11 +388,14 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise { reject(new SubagentTaskOwnerUnavailableError()); }, DEFAULT_READY_TIMEOUT_MS); timeout.unref?.(); + signal.addEventListener('abort', onEnd, { once: true }); client.once('ready', onReady); client.once('end', onEnd); /** Close the status-check/listener-registration race: ioredis may become ready * synchronously between the check above and installing these listeners. */ - if (client.status === 'ready') { + if (signal.aborted) { + onEnd(); + } else if (client.status === 'ready') { onReady(); } else if (client.status === 'end') { onEnd(); @@ -397,9 +407,10 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise { async function waitForRedisReady( client: RedisClient, + signal: AbortSignal, options: { eagerClusterMasters?: boolean } = {}, ): Promise { - await waitForRedisConnectionReady(client); + await waitForRedisConnectionReady(client, signal); if (options.eagerClusterMasters !== true || !client.isCluster) { return; } @@ -407,10 +418,41 @@ async function waitForRedisReady( * fail-fast publisher cannot admit requests until every possible write target is * connected; otherwise the first command to a cold shard would be rejected. */ await Promise.all( - (client as Cluster).nodes('master').map((node) => waitForRedisConnectionReady(node)), + (client as Cluster).nodes('master').map((node) => waitForRedisConnectionReady(node, signal)), ); } +/** Bounds the initial SUBSCRIBE acknowledgement and cancels it with its owner. */ +async function waitForRedisSubscription( + operation: Promise, + signal: AbortSignal, +): Promise { + await new Promise((resolve, reject) => { + const cleanup = (): void => { + clearTimeout(timeout); + signal.removeEventListener('abort', onAbort); + }; + const onAbort = (): void => { + cleanup(); + reject(new SubagentTaskOwnerUnavailableError()); + }; + const timeout = setTimeout(onAbort, DEFAULT_READY_TIMEOUT_MS); + timeout.unref?.(); + signal.addEventListener('abort', onAbort, { once: true }); + operation.then( + () => { + cleanup(); + resolve(); + }, + (error: unknown) => { + cleanup(); + reject(error); + }, + ); + if (signal.aborted) onAbort(); + }); +} + function isSnapshot(value: unknown): value is SubagentTaskSnapshot { if (value == null || typeof value !== 'object' || Array.isArray(value)) { return false; @@ -734,7 +776,9 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra private registrationHeartbeat?: ReturnType; private registrationRefresh?: Promise; private destroyed = false; + private readonly lifecycle = new AbortController(); + /** Borrows the publisher; owns the dedicated subscriber and its pending waits. */ constructor( private readonly publisher: RedisClient, private readonly subscriber: RedisClient, @@ -764,13 +808,22 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra /** The publisher fails fast instead of queueing commands, so opening HTTP * admission before it is ready would turn healthy startup lag into false * `unavailable` results. Both dedicated connections are part of readiness. */ - await Promise.all([ - waitForRedisReady(this.publisher, { eagerClusterMasters: true }), - waitForRedisReady(this.subscriber), - ]); - this.subscriber.on('message', this.onMessage); - this.ready = this.subscriber.subscribe(this.channel(this.instanceId)).then(() => undefined); - await this.ready; + try { + await Promise.all([ + waitForRedisReady(this.publisher, this.lifecycle.signal, { eagerClusterMasters: true }), + waitForRedisReady(this.subscriber, this.lifecycle.signal), + ]); + if (this.destroyed) throw new SubagentTaskOwnerUnavailableError(); + this.subscriber.on('message', this.onMessage); + this.ready = waitForRedisSubscription( + this.subscriber.subscribe(this.channel(this.instanceId)), + this.lifecycle.signal, + ); + await this.requireReady(); + } catch (error) { + await this.destroy(); + throw error; + } } async registerTask(scopeId: string, taskId: string, ttlMs: number): Promise { @@ -963,6 +1016,8 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra return; } this.destroyed = true; + this.lifecycle.abort(); + this.handler = undefined; for (const pending of this.pending.values()) { clearTimeout(pending.retry); clearTimeout(pending.timeout); @@ -979,12 +1034,15 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra this.registrationHeartbeat = undefined; } this.subscriber.off('message', this.onMessage); - await this.subscriber.unsubscribe(this.channel(this.instanceId)).catch(() => undefined); this.subscriber.disconnect(); } private readonly onMessage = (channel: string, message: string): void => { - if (channel !== this.channel(this.instanceId) || message.length > MAX_ROUTED_MESSAGE_CHARS) { + if ( + this.destroyed || + channel !== this.channel(this.instanceId) || + message.length > MAX_ROUTED_MESSAGE_CHARS + ) { return; } let parsed: unknown; @@ -1027,6 +1085,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra } private async handleRequest(request: RoutedRequest): Promise { + if (this.destroyed) return; if (Date.now() > request.expiresAt + REQUEST_CLOCK_SKEW_MS) { /** The caller stopped waiting for this long ago and has been told it was * unavailable, so applying it now would steer a child it believes untouched. */ @@ -1094,6 +1153,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra ), ); } + if (this.destroyed) return; const serializedResult = JSON.stringify(result); /** Retaining the result rather than the envelope lets a later caller retry, * which carries its own correlation id, recover a response it never received. */ @@ -1102,6 +1162,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra } serialized = successResponse(request.requestId, serializedResult); } catch (error) { + if (this.destroyed) return; logger.error('[subagentTaskRouting] Owner failed to process a routed command', error); serialized = failureResponse(request.requestId); } @@ -1306,6 +1367,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra } private async publish(channel: string, value: string): Promise { + if (this.destroyed) throw new SubagentTaskOwnerUnavailableError(); const delivered = await this.publisher.publish(channel, value); return typeof delivered === 'number' ? delivered : 0; } @@ -1451,5 +1513,6 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra throw new SubagentTaskOwnerUnavailableError(); } await this.ready; + if (this.destroyed) throw new SubagentTaskOwnerUnavailableError(); } } diff --git a/packages/api/src/app/invalidation.redis.spec.ts b/packages/api/src/app/invalidation.redis.spec.ts index 170df2e796..792616511a 100644 --- a/packages/api/src/app/invalidation.redis.spec.ts +++ b/packages/api/src/app/invalidation.redis.spec.ts @@ -25,10 +25,8 @@ describe('createRedisConfigInvalidationPubSub', () => { }); afterEach(async () => { - for (const { invalidation, pubsub } of opened) { + for (const { invalidation } of opened) { await invalidation.close().catch(() => undefined); - (pubsub.publisher as Redis).disconnect(); - (pubsub.subscriber as Redis).disconnect(); } for (const client of clients) { client.disconnect(); @@ -196,8 +194,7 @@ describe('createRedisConfigInvalidationPubSub', () => { server = await startRespServer(); }); - it('subscribes once Redis comes back when the first SUBSCRIBE failed during an outage', async () => { - const error = jest.spyOn(logger, 'error').mockReturnValue(logger); + it('subscribes once Redis comes back when the first SUBSCRIBE is pending during an outage', async () => { const channel = 'librechat:config:invalidate:deploy-a'; const client = await connect(); const { port } = server; @@ -205,13 +202,9 @@ describe('createRedisConfigInvalidationPubSub', () => { await waitFor(() => client.status !== 'ready'); const late = await startInstanceOn(client, 'deploy-a'); - await waitFor( - () => - error.mock.calls.some(([message]) => - String(message).includes('Subscribing to cross-instance invalidations failed'), - ), - 5000, - ); + const subscribe = jest.spyOn(late.pubsub.subscriber, 'subscribe'); + await waitFor(() => (late.pubsub.subscriber as Redis).status === 'reconnecting'); + expect(server.commands.filter((frame) => frame[0]?.toUpperCase() === 'SUBSCRIBE')).toEqual([]); server = await startRespServer(port); await waitFor(subscribed(channel, 1), 5000); @@ -222,6 +215,48 @@ describe('createRedisConfigInvalidationPubSub', () => { await waitFor(() => late.clearOverrides.mock.calls.length === 1); expect(late.clearOverrides).toHaveBeenCalledWith('tenant-a'); + expect(subscribe).not.toHaveBeenCalled(); + }); + + it.each([false, true])( + 'closes its owned connections during an outage (subscription acknowledged=%s)', + async (acknowledged) => { + const client = await connect(); + const early = acknowledged ? await startInstanceOn(client, 'deploy-a') : undefined; + if (early) { + await waitFor(() => (early.pubsub.subscriber as Redis).mode === 'subscriber'); + } + await server.close(); + await waitFor(() => client.status !== 'ready'); + const instance = early ?? (await startInstanceOn(client, 'deploy-a')); + const subscriber = instance.pubsub.subscriber as Redis; + const publisher = instance.pubsub.publisher as Redis; + await waitFor(() => subscriber.status === 'reconnecting'); + const disconnectSubscriber = jest.spyOn(subscriber, 'disconnect'); + const disconnectPublisher = jest.spyOn(publisher, 'disconnect'); + const unsubscribe = jest.spyOn(subscriber, 'unsubscribe'); + + const started = Date.now(); + await instance.invalidation.close(); + await instance.invalidation.close(); + + expect(Date.now() - started).toBeLessThan(1000); + expect(disconnectSubscriber).toHaveBeenCalledTimes(1); + expect(disconnectPublisher).toHaveBeenCalledTimes(1); + expect(unsubscribe).not.toHaveBeenCalled(); + }, + ); + + it('leaves the shared client usable when the owned pair is closed', async () => { + const client = await connect(); + const instance = await startInstanceOn(client, 'deploy-a'); + await waitFor(() => (instance.pubsub.subscriber as Redis).mode === 'subscriber'); + + await instance.invalidation.close(); + + await expect(client.ping()).resolves.toBe('PONG'); + await waitFor(() => (instance.pubsub.subscriber as Redis).status === 'end'); + await waitFor(() => (instance.pubsub.publisher as Redis).status === 'end'); }); it('returns no pub/sub without Redis', async () => { diff --git a/packages/api/src/app/invalidation.spec.ts b/packages/api/src/app/invalidation.spec.ts index 6f671d3ed1..12213b732d 100644 --- a/packages/api/src/app/invalidation.spec.ts +++ b/packages/api/src/app/invalidation.spec.ts @@ -183,6 +183,75 @@ describe('createConfigCacheInvalidation', () => { expect(second.clearOverrides).not.toHaveBeenCalled(); }); + it('releases borrowed listeners without waiting for an unavailable unsubscribe acknowledgement', async () => { + const broker = createBroker(); + const subscriber = broker.subscriber(); + jest.spyOn(subscriber, 'unsubscribe').mockImplementation(() => new Promise(() => undefined)); + const invalidation = createConfigCacheInvalidation({ + clears: [], + pubsub: { publisher: broker.publisher(), subscriber }, + }); + let closed = false; + + void invalidation.close().then(() => { + closed = true; + }); + await flush(); + + expect(closed).toBe(true); + expect(subscriber.events.listenerCount('message')).toBe(0); + expect(subscriber.events.listenerCount('ready')).toBe(0); + }); + + it('releases an owned pub/sub pair once instead of awaiting unsubscribe', async () => { + const broker = createBroker(); + const subscriber = broker.subscriber(); + const unsubscribe = jest.spyOn(subscriber, 'unsubscribe'); + const close = jest.fn(); + const invalidation = createConfigCacheInvalidation({ + clears: [], + pubsub: { publisher: broker.publisher(), subscriber, close }, + }); + await flush(); + + await invalidation.close(); + await invalidation.close(); + subscriber.events.emit('ready'); + await flush(); + + expect(close).toHaveBeenCalledTimes(1); + expect(unsubscribe).not.toHaveBeenCalled(); + expect(subscriber.subscribe).toHaveBeenCalledTimes(1); + expect(subscriber.events.listenerCount('ready')).toBe(0); + expect(subscriber.events.listenerCount('message')).toBe(0); + }); + + it('does not report a pending subscription rejected by intentional close as an outage', async () => { + const error = jest.spyOn(logger, 'error').mockReturnValue(logger); + const broker = createBroker(); + const subscriber = broker.subscriber(); + let rejectSubscribe: (error: Error) => void = () => undefined; + subscriber.subscribe.mockImplementation( + () => + new Promise((_resolve, reject) => { + rejectSubscribe = reject; + }), + ); + const invalidation = createConfigCacheInvalidation({ + clears: [], + pubsub: { + publisher: broker.publisher(), + subscriber, + close: () => rejectSubscribe(new Error('Connection is closed.')), + }, + }); + + await invalidation.close(); + await flush(); + + expect(error).not.toHaveBeenCalled(); + }); + it('subscribes again when the subscriber becomes ready after a failed SUBSCRIBE', async () => { const error = jest.spyOn(logger, 'error').mockReturnValue(logger); const broker = createBroker(); diff --git a/packages/api/src/app/invalidation.ts b/packages/api/src/app/invalidation.ts index 21ed04bfe1..e3dbbc1c40 100644 --- a/packages/api/src/app/invalidation.ts +++ b/packages/api/src/app/invalidation.ts @@ -31,6 +31,8 @@ export interface ConfigInvalidationPubSub { publisher: ConfigInvalidationPublisher; subscriber: ConfigInvalidationSubscriber; channel?: string; + /** Releases owned connections without waiting for a Redis acknowledgement. */ + close?(): void; } /** One cache a config write invalidates; `tenantId` scopes it when the cache supports that. */ @@ -158,6 +160,9 @@ export function createConfigCacheInvalidation(params: { }, (error: unknown) => { subscribing = false; + if (closed) { + return; + } logger.error( '[invalidateConfigCaches] Subscribing to cross-instance invalidations failed; retrying when the subscriber reconnects. Until then this instance may serve cached config until it expires:', getSafeErrorMetadata(error), @@ -209,13 +214,17 @@ export function createConfigCacheInvalidation(params: { } async function close(): Promise { - if (!pubsub) { + if (!pubsub || closed) { return; } closed = true; pubsub.subscriber.off('ready', subscribe); pubsub.subscriber.off('message', onMessage); - await pubsub.subscriber.unsubscribe(channel); + if (pubsub.close) { + pubsub.close(); + return; + } + void pubsub.subscriber.unsubscribe(channel).catch(() => undefined); } return { invalidate, reload, close }; @@ -242,9 +251,14 @@ export function createRedisConfigInvalidationPubSub(params: { publisher.on('error', (error: Error) => { logger.error('[ConfigInvalidation] publisher error:', getSafeErrorMetadata(error)); }); + const subscriber = createIoRedisSubscriber(client, '[ConfigInvalidation] subscriber'); return { publisher, - subscriber: createIoRedisSubscriber(client, '[ConfigInvalidation] subscriber'), + subscriber, channel: `${CONFIG_INVALIDATION_CHANNEL}:${namespace?.trim() || 'default'}`, + close: () => { + subscriber.disconnect(); + publisher.disconnect(); + }, }; } diff --git a/packages/api/src/cache/__tests__/outage.cache_integration.spec.ts b/packages/api/src/cache/__tests__/outage.cache_integration.spec.ts new file mode 100644 index 0000000000..bdd3de6649 --- /dev/null +++ b/packages/api/src/cache/__tests__/outage.cache_integration.spec.ts @@ -0,0 +1,217 @@ +import type { Redis, Cluster } from 'ioredis'; +import type { RedisEventTransport } from '~/stream/implementations/RedisEventTransport'; +import type { RedisClientsModule } from './redisClients.helper'; +import { startRedis, sleep, waitUntil } from './outage.helper'; +import { isRedisReady } from '~/app/readiness'; + +jest.mock('@librechat/data-schemas', () => ({ + logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() }, +})); +jest.mock('~/utils', () => ({ + ...jest.requireActual('~/utils/math'), + ...jest.requireActual('~/utils/common'), +})); + +describe.each([false, true])('Redis outage recovery (cluster=%s)', (cluster) => { + let server: Awaited> | undefined; + let clients: RedisClientsModule | undefined; + let subscriber: Redis | Cluster | undefined; + let env: NodeJS.ProcessEnv; + const transports: RedisEventTransport[] = []; + const extraClients: (Redis | Cluster)[] = []; + + const ready = (): boolean => + isRedisReady({ + useRedis: true, + ioredis: clients?.ioredisClient ?? null, + keyv: clients?.keyvRedisClient ?? null, + }); + + const connect = async (): Promise => { + clients = await import('../redisClients'); + await clients.waitForKeyvRedisClient(); + await waitUntil(ready); + }; + + beforeEach(async () => { + env = { ...process.env }; + server = await startRedis(cluster); + Object.assign(process.env, { + USE_REDIS: 'true', + USE_REDIS_CLUSTER: String(cluster), + REDIS_URI: server.url, + REDIS_RETRY_MAX_DELAY: '100', + REDIS_PING_INTERVAL: '0', + REDIS_SUBSCRIBER_PING_INTERVAL: '0', + REDIS_KEY_PREFIX: '', + REDIS_KEY_PREFIX_VAR: '', + REDIS_USERNAME: '', + REDIS_PASSWORD: '', + REDIS_CA: '', + REDIS_ENABLE_OFFLINE_QUEUE: 'true', + }); + delete process.env.REDIS_RETRY_MAX_ATTEMPTS; + jest.resetModules(); + }); + + afterEach(async () => { + for (const transport of transports.splice(0)) { + transport.destroy(); + } + for (const client of extraClients.splice(0)) { + client.disconnect(); + } + subscriber?.disconnect(); + subscriber = undefined; + clients?.ioredisClient?.disconnect(); + const keyv = clients?.keyvRedisClient; + try { + if (keyv?.isOpen) { + keyv.destroy(); + } + } catch (error) { + // Cluster teardown can encounter a master that exhausted its finite budget. + if (!(error instanceof Error) || error.message !== 'The client is closed') { + throw error; + } + } finally { + clients = undefined; + try { + await server?.close(); + } finally { + server = undefined; + process.env = env; + } + } + }); + + it('recovers both clients and subscriptions after 20 seconds without replaying an uncertain write', async () => { + await connect(); + const io = clients!.ioredisClient!; + const keyv = clients!.keyvRedisClient!; + const { createIoRedisSubscriber } = await import('../redisUtils'); + subscriber = createIoRedisSubscriber(io, 'outage-subscriber'); + const messages: string[] = []; + subscriber.on('message', (_channel: string, value: string) => messages.push(value)); + await waitUntil(() => subscriber?.status === 'ready'); + await subscriber.subscribe('outage-events'); + await io.publish('outage-events', 'before'); + await waitUntil(() => messages.includes('before')); + + server!.dropReplies(); + const pendingWrite = io.incr('outage-counter').then( + () => 'resolved', + () => 'rejected', + ); + const pendingKeyvWrite = keyv.incr('keyv-outage-counter').then( + () => 'resolved', + () => 'rejected', + ); + let applied = false; + const deadline = Date.now() + 5000; + while (!applied && Date.now() < deadline) { + applied = + (await server!.admin.get('::outage-counter')) === '1' && + (await server!.admin.get('keyv-outage-counter')) === '1'; + if (!applied) await sleep(20); + } + expect(applied).toBe(true); + server!.block(); + await waitUntil(() => !ready()); + await sleep(20_000); + expect(ready()).toBe(false); + await expect(pendingWrite).resolves.toBe('rejected'); + await expect(pendingKeyvWrite).resolves.toBe('rejected'); + + server!.restore(); + await waitUntil(ready); + await waitUntil(() => subscriber?.status === 'ready'); + await expect(io.set('after-outage', 'ioredis')).resolves.toBe('OK'); + await expect(io.get('after-outage')).resolves.toBe('ioredis'); + await expect(keyv.set('keyv-after-outage', 'keyv')).resolves.toBe('OK'); + await expect(keyv.get('keyv-after-outage')).resolves.toBe('keyv'); + await expect(server!.admin.get('::outage-counter')).resolves.toBe('1'); + await expect(server!.admin.get('keyv-outage-counter')).resolves.toBe('1'); + await io.publish('outage-events', 'after'); + await waitUntil(() => messages.includes('after')); + expect(messages).toEqual(['before', 'after']); + }, 40_000); + + it('keeps stream and abort subscriptions after an interrupted resubscribe acknowledgement', async () => { + await connect(); + const { createIoRedisSubscriber, duplicateIoRedisClient } = await import('../redisUtils'); + const { RedisEventTransport } = await import('~/stream/implementations/RedisEventTransport'); + const io = clients!.ioredisClient!; + subscriber = createIoRedisSubscriber(io, 'outage-owner'); + let readyEvents = 0; + subscriber.on('ready', () => readyEvents++); + const publisher = duplicateIoRedisClient(io); + extraClients.push(publisher); + const owner = new RedisEventTransport(io, subscriber); + const peer = new RedisEventTransport(publisher, createIoRedisSubscriber(io, 'outage-peer')); + transports.push(owner, peer); + const streamId = 'interrupted-subscription'; + const channel = `stream:{${streamId}}:events`; + const chunks: unknown[] = []; + let abortedGeneration: number | undefined; + const attachment = owner.subscribe(streamId, { onChunk: (event) => chunks.push(event) }); + await attachment.ready; + const unsubscribeAbort = await owner.onAbort(streamId, (generation) => { + abortedGeneration = generation; + }); + await waitUntil(() => publisher.status === 'ready'); + await peer.emitChunk(streamId, { value: 'before' }); + await waitUntil(() => chunks.length === 1); + + server!.block(); + await waitUntil(() => !ready()); + server!.interruptNextSubscription(); + server!.restore(); + await waitUntil(() => server!.interruptedSubscriptions === 1); + await waitUntil( + () => ready() && subscriber?.status === 'ready' && readyEvents >= (cluster ? 2 : 3), + ); + + const isSubscribed = async (): Promise => { + const counts = await server!.admin.pubsub('NUMSUB', channel); + return Array.isArray(counts) && counts[1] === 1; + }; + let subscribed = await isSubscribed(); + const deadline = Date.now() + 2000; + while (!subscribed && Date.now() < deadline) { + await sleep(20); + subscribed = await isSubscribed(); + } + expect(subscribed).toBe(true); + await peer.emitChunk(streamId, { value: 'after' }); + peer.emitAbort(streamId, 7); + await waitUntil(() => chunks.length === 2 && abortedGeneration === 7); + expect(chunks).toEqual([{ value: 'before' }, { value: 'after' }]); + unsubscribeAbort(); + attachment.unsubscribe(); + }, 15_000); + + it('honors an explicit finite reconnect budget and remains not ready', async () => { + process.env.REDIS_RETRY_MAX_ATTEMPTS = '2'; + await connect(); + const { createIoRedisSubscriber } = await import('../redisUtils'); + subscriber = createIoRedisSubscriber(clients!.ioredisClient!, 'finite-subscriber'); + await waitUntil(() => subscriber?.status === 'ready'); + await subscriber.subscribe('finite-events'); + server!.block(); + await waitUntil(() => !ready()); + await sleep(2000); + server!.restore(); + await sleep(500); + expect(ready()).toBe(false); + expect(clients!.ioredisClient!.status).not.toBe('ready'); + expect(subscriber.status).not.toBe('ready'); + expect( + isRedisReady({ + useRedis: true, + ioredis: { status: 'ready' }, + keyv: clients!.keyvRedisClient, + }), + ).toBe(false); + }, 15_000); +}); diff --git a/packages/api/src/cache/__tests__/outage.helper.ts b/packages/api/src/cache/__tests__/outage.helper.ts new file mode 100644 index 0000000000..129a612b5f --- /dev/null +++ b/packages/api/src/cache/__tests__/outage.helper.ts @@ -0,0 +1,220 @@ +import net from 'node:net'; +import Redis from 'ioredis'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { once } from 'node:events'; +import { spawn } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import type { ChildProcess } from 'node:child_process'; +import type { Socket } from 'node:net'; + +export const sleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +export async function waitUntil(predicate: () => boolean, timeout = 5000): Promise { + const deadline = Date.now() + timeout; + while (!predicate()) { + if (Date.now() >= deadline) { + throw new Error('Timed out waiting for Redis state'); + } + await sleep(20); + } +} + +async function freePort(): Promise { + const server = net.createServer(); + try { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected a TCP address'); + } + return address.port; + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +} + +/** Real Redis behind a TCP fault boundary, including the port advertised to Cluster clients. */ +export async function startRedis(cluster: boolean) { + const port = await freePort(); + let busPort = await freePort(); + while (busPort === port) { + busPort = await freePort(); + } + const sockets = new Set(); + let blocked = false; + let dropReplies = false; + let interruptSubscription = false; + let interruptedSubscriptions = 0; + const proxy = net.createServer((socket) => { + if (blocked) { + socket.destroy(); + return; + } + const upstream = net.connect(port, '127.0.0.1'); + for (const connection of [socket, upstream]) { + sockets.add(connection); + connection.on('error', () => connection.destroy()); + connection.on('close', () => { + sockets.delete(connection); + socket.destroy(); + upstream.destroy(); + }); + } + socket.pipe(upstream); + let replyTail = ''; + const subscribeReply = '$9\r\nsubscribe\r\n'; + upstream.on('data', (chunk: Buffer) => { + const reply = replyTail + chunk.toString('latin1'); + replyTail = reply.slice(-subscribeReply.length); + if (interruptSubscription && reply.includes(subscribeReply)) { + interruptSubscription = false; + interruptedSubscriptions++; + socket.destroy(); + upstream.destroy(); + return; + } + if (!dropReplies) { + socket.write(chunk); + } + }); + }); + let dir: string | undefined; + let child: ChildProcess | undefined; + let childClosed: Promise | undefined; + let admin: Redis | undefined; + let startupError: Error | undefined; + const close = async (): Promise => { + admin?.disconnect(); + for (const socket of sockets) { + socket.destroy(); + } + const proxyClosed = new Promise((resolve) => proxy.close(() => resolve())); + if (child?.pid && child.exitCode === null && child.signalCode === null) { + child.kill('SIGTERM'); + } + await Promise.all([proxyClosed, childClosed]); + if (dir) { + await rm(dir, { recursive: true, force: true }); + } + }; + + try { + proxy.listen(0, '127.0.0.1'); + await once(proxy, 'listening'); + const address = proxy.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected a TCP proxy address'); + } + dir = await mkdtemp(join(tmpdir(), 'redis-outage-')); + const binary = process.env.REDIS_SERVER_BIN ?? 'redis-server'; + child = spawn( + binary, + [ + '--bind', + '127.0.0.1', + '--port', + String(port), + '--dir', + dir, + '--save', + '', + '--appendonly', + 'no', + '--unixsocket', + '', + ...(cluster + ? [ + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-port', + String(busPort), + '--cluster-announce-ip', + '127.0.0.1', + '--cluster-announce-port', + String(address.port), + '--cluster-announce-bus-port', + String(busPort), + ] + : []), + ], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stderr = ''; + child.stdout?.resume(); + child.stderr?.on('data', (chunk: Buffer) => { + stderr = (stderr + chunk.toString()).slice(-4096); + }); + child.on('error', (error: Error) => { + startupError = new Error(`Unable to start Redis (${binary}): ${error.message}`, { + cause: error, + }); + }); + childClosed = new Promise((resolve) => { + child!.once('close', (code, signal) => { + startupError ??= new Error( + `Redis exited before becoming ready (code=${code}, signal=${signal}): ${stderr.trim()}`, + ); + resolve(); + }); + }); + admin = new Redis(port, '127.0.0.1', { + retryStrategy: () => 20, + enableReadyCheck: false, + maxRetriesPerRequest: 3, + keyPrefix: '', + }); + admin.on('error', () => undefined); + await waitUntil(() => { + if (startupError) { + throw startupError; + } + return admin?.status === 'ready'; + }); + if (cluster) { + await admin.cluster('ADDSLOTSRANGE', '0', '16383'); + const deadline = Date.now() + 5000; + while (!((await admin.cluster('INFO')) as string).includes('cluster_state:ok')) { + if (startupError) { + throw startupError; + } + if (Date.now() > deadline) { + throw new Error('Redis cluster did not become ready'); + } + await sleep(50); + } + } + return { + url: `redis://127.0.0.1:${address.port}`, + admin, + dropReplies: () => { + dropReplies = true; + }, + interruptNextSubscription: () => { + interruptSubscription = true; + }, + get interruptedSubscriptions() { + return interruptedSubscriptions; + }, + block: () => { + blocked = true; + for (const socket of sockets) { + socket.destroy(); + } + }, + restore: () => { + blocked = false; + dropReplies = false; + }, + close, + }; + } catch (error) { + const failure = startupError ?? error; + await close(); + throw failure; + } +} diff --git a/packages/api/src/cache/__tests__/outage.spec.ts b/packages/api/src/cache/__tests__/outage.spec.ts new file mode 100644 index 0000000000..a0dfcf159a --- /dev/null +++ b/packages/api/src/cache/__tests__/outage.spec.ts @@ -0,0 +1,97 @@ +import net from 'node:net'; +import Redis from 'ioredis'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import fs from 'node:fs/promises'; +import childProcess from 'node:child_process'; +import { startRedis } from './outage.helper'; + +describe.each([false, true])('Redis outage fixture startup (cluster=%s)', (cluster) => { + let env: NodeJS.ProcessEnv; + let dir: string; + let servers: jest.SpiedFunction; + let directories: jest.SpiedFunction; + let disconnect: jest.SpiedFunction; + let spawned: jest.SpiedFunction; + + beforeAll(async () => { + dir = await fs.mkdtemp(join(tmpdir(), 'redis-fixture-test-')); + await fs.writeFile( + join(dir, 'exits'), + `#!${process.execPath}\nprocess.stderr.write('fixture startup refused\\n', () => process.exit(23));\n`, + { mode: 0o755 }, + ); + await fs.writeFile( + join(dir, 'idle'), + `#!${process.execPath}\nsetInterval(() => undefined, 1000);\n`, + { mode: 0o755 }, + ); + }); + + beforeEach(() => { + env = { ...process.env }; + servers = jest.spyOn(net, 'createServer'); + directories = jest.spyOn(fs, 'mkdtemp'); + disconnect = jest.spyOn(Redis.prototype, 'disconnect'); + spawned = jest.spyOn(childProcess, 'spawn'); + }); + + afterEach(() => { + process.env = env; + jest.restoreAllMocks(); + }); + + afterAll(async () => { + await fs.rm(dir, { recursive: true, force: true }); + }); + + const expectServersClosed = (): void => { + expect(servers).toHaveBeenCalledTimes(3); + for (const result of servers.mock.results) { + if (result.type === 'return') { + expect(result.value.listening).toBe(false); + } + } + }; + + const expectCleanup = async (): Promise => { + expectServersClosed(); + expect(disconnect).toHaveBeenCalledTimes(1); + expect(directories).toHaveBeenCalledTimes(1); + const directory: string = await directories.mock.results[0].value; + await expect(fs.access(directory)).rejects.toMatchObject({ code: 'ENOENT' }); + }; + + it('rejects a missing binary promptly and closes its proxy, admin, and directory', async () => { + const binary = join(dir, 'missing-redis-server'); + process.env.REDIS_SERVER_BIN = binary; + await expect(startRedis(cluster)).rejects.toThrow( + `Unable to start Redis (${binary}): spawn ${binary} ENOENT`, + ); + await expectCleanup(); + }, 3000); + + it('reports an early exit with its code and stderr, then cleans up', async () => { + process.env.REDIS_SERVER_BIN = join(dir, 'exits'); + await expect(startRedis(cluster)).rejects.toThrow( + 'Redis exited before becoming ready (code=23, signal=null): fixture startup refused', + ); + await expectCleanup(); + }, 3000); + + it('terminates a live child and removes its resources when readiness times out', async () => { + process.env.REDIS_SERVER_BIN = join(dir, 'idle'); + await expect(startRedis(cluster)).rejects.toThrow('Timed out waiting for Redis state'); + expect(spawned).toHaveBeenCalledTimes(1); + expect(spawned.mock.results[0].value.signalCode).toBe('SIGTERM'); + await expectCleanup(); + }); + + it('closes the proxy if allocating the Redis directory fails', async () => { + const error = new Error('fixture directory unavailable'); + directories.mockRejectedValueOnce(error); + await expect(startRedis(cluster)).rejects.toBe(error); + expectServersClosed(); + expect(disconnect).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/cache/cacheConfig.spec.ts b/packages/api/src/cache/cacheConfig.spec.ts new file mode 100644 index 0000000000..bdc50df9d7 --- /dev/null +++ b/packages/api/src/cache/cacheConfig.spec.ts @@ -0,0 +1,33 @@ +jest.mock('@librechat/data-schemas', () => ({ logger: { warn: jest.fn(), error: jest.fn() } })); +jest.mock('~/utils', () => ({ + ...jest.requireActual('~/utils/math'), + ...jest.requireActual('~/utils/common'), +})); + +describe('Redis retry configuration', () => { + const original = process.env.REDIS_RETRY_MAX_ATTEMPTS; + + afterEach(() => { + if (original === undefined) { + delete process.env.REDIS_RETRY_MAX_ATTEMPTS; + } else { + process.env.REDIS_RETRY_MAX_ATTEMPTS = original; + } + }); + + it.each([ + [undefined, 0], + ['0', 0], + ['5', 5], + ])('reads %s as %s attempts', (value, expected) => { + if (value === undefined) { + delete process.env.REDIS_RETRY_MAX_ATTEMPTS; + } else { + process.env.REDIS_RETRY_MAX_ATTEMPTS = value; + } + jest.isolateModules(() => { + const { cacheConfig } = jest.requireActual('./cacheConfig'); + expect(cacheConfig.REDIS_RETRY_MAX_ATTEMPTS).toBe(expected); + }); + }); +}); diff --git a/packages/api/src/cache/cacheConfig.ts b/packages/api/src/cache/cacheConfig.ts index 61672c283c..9664d6a027 100644 --- a/packages/api/src/cache/cacheConfig.ts +++ b/packages/api/src/cache/cacheConfig.ts @@ -174,7 +174,7 @@ const cacheConfig: { /** Max delay between reconnection attempts in ms */ REDIS_RETRY_MAX_DELAY: math(process.env.REDIS_RETRY_MAX_DELAY, 3000), /** Max number of reconnection attempts (0 = infinite) */ - REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 10), + REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 0), /** Connection timeout in ms */ REDIS_CONNECT_TIMEOUT: math(process.env.REDIS_CONNECT_TIMEOUT, 10000), /** Min spacing in ms between reconnects forced by READONLY replies after a failover */ diff --git a/packages/api/src/cache/redisClients.ts b/packages/api/src/cache/redisClients.ts index ebe87b7879..374a3e1514 100644 --- a/packages/api/src/cache/redisClients.ts +++ b/packages/api/src/cache/redisClients.ts @@ -7,6 +7,7 @@ import type { RedisClientType, RedisClusterType } from '@redis/client'; import type { Redis, Cluster } from 'ioredis'; import type { ReadonlyRecoveryHandler } from './recovery'; import { createReadonlyRecovery, isReadonlyReplicaError } from './recovery'; +import { createRedisRetryStrategy } from './retry'; import { startRedisHeartbeat } from './heartbeat'; import { cacheConfig } from './cacheConfig'; @@ -48,22 +49,13 @@ if (cacheConfig.USE_REDIS) { tls: useTls ? { ca: ca ?? undefined } : undefined, keyPrefix: `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR}`, maxListeners: cacheConfig.REDIS_MAX_LISTENERS, - retryStrategy: (times: number) => { - if ( - cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 && - times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS - ) { - logger.error( - `ioredis giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`, - ); - return null; - } - const base = Math.min(Math.pow(2, times) * 50, cacheConfig.REDIS_RETRY_MAX_DELAY); - const jitter = Math.floor(Math.random() * Math.min(base, 1000)); - const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY); - logger.info(`ioredis reconnecting... attempt ${times}, delay ${delay}ms`); - return delay; - }, + retryStrategy: createRedisRetryStrategy({ + label: 'ioredis', + maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS, + maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY, + initialDelay: 50, + log: (message) => logger.error(message), + }), reconnectOnError: (err: Error) => { if (isReadonlyReplicaError(err)) { logger.warn('ioredis reconnecting due to READONLY error'); @@ -73,7 +65,9 @@ if (cacheConfig.USE_REDIS) { }, enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE, connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT, - maxRetriesPerRequest: 3, + // A lost reply does not prove a write failed. Reconnect without replaying it. + maxRetriesPerRequest: 0, + autoResendUnfulfilledCommands: false, keepAlive: cacheConfig.REDIS_KEEP_ALIVE, }; @@ -91,32 +85,25 @@ if (cacheConfig.USE_REDIS) { } : {}), redisOptions, - clusterRetryStrategy: (times: number) => { - if ( - cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 && - times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS - ) { - logger.error( - `ioredis cluster giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`, - ); - return null; - } - const base = Math.min(Math.pow(2, times) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY); - const jitter = Math.floor(Math.random() * Math.min(base, 1000)); - const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY); - logger.info(`ioredis cluster reconnecting... attempt ${times}, delay ${delay}ms`); - return delay; - }, + // Keep MOVED/ASK routing, but never resend a command after losing its reply. + retryDelayOnFailover: 0, + clusterRetryStrategy: createRedisRetryStrategy({ + label: 'ioredis cluster', + maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS, + maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY, + initialDelay: 100, + log: (message) => logger.error(message), + }), enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE, }, ); ioredisClient.on('error', (err) => { - logger.error('ioredis client error:', err); + logger.debug('ioredis client error:', err); }); ioredisClient.on('connect', () => { - logger.info('ioredis client connected'); + logger.debug('ioredis client connected'); }); ioredisClient.on('ready', () => { @@ -124,11 +111,11 @@ if (cacheConfig.USE_REDIS) { }); ioredisClient.on('reconnecting', (delay: number) => { - logger.info(`ioredis client reconnecting in ${delay}ms`); + logger.debug(`ioredis client reconnecting in ${delay}ms`); }); ioredisClient.on('close', () => { - logger.warn('ioredis client connection closed'); + logger.debug('ioredis client connection closed'); }); /** Deadline-bounded keepalive PINGs; an unanswered probe forces a reconnect. */ @@ -201,6 +188,14 @@ if (cacheConfig.USE_REDIS) { * ** WARNING ** Keyv Redis client does not support Prefix like ioredis above. * The prefix feature will be handled by the Keyv-Redis store in cacheFactory.js */ + const retry = createRedisRetryStrategy({ + label: '@keyv/redis client', + maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS, + maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY, + initialDelay: 100, + firstAttempt: 0, + log: (message) => logger.error(message), + }); const redisOptions: Record = { username, password, @@ -208,22 +203,8 @@ if (cacheConfig.USE_REDIS) { ...(isRedisCluster ? { tls: useTls } : {}), ...(ca ? { ca } : {}), connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT, - reconnectStrategy: (retries: number) => { - if ( - cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 && - retries > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS - ) { - logger.error( - `@keyv/redis client giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`, - ); - return new Error('Max reconnection attempts reached'); - } - const base = Math.min(Math.pow(2, retries) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY); - const jitter = Math.floor(Math.random() * Math.min(base, 1000)); - const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY); - logger.info(`@keyv/redis reconnecting... attempt ${retries}, delay ${delay}ms`); - return delay; - }, + reconnectStrategy: (retries: number) => + retry(retries) ?? new Error('Max reconnection attempts reached'), }, disableOfflineQueue: !cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE, ...(cacheConfig.REDIS_PING_INTERVAL > 0 @@ -264,11 +245,11 @@ if (cacheConfig.USE_REDIS) { } keyvRedisClient.on('error', (err) => { - logger.error('@keyv/redis client error:', err); + logger.debug('@keyv/redis client error:', err); }); keyvRedisClient.on('connect', () => { - logger.info('@keyv/redis client connected'); + logger.debug('@keyv/redis client connected'); }); keyvRedisClient.on('ready', () => { @@ -276,11 +257,11 @@ if (cacheConfig.USE_REDIS) { }); keyvRedisClient.on('reconnecting', () => { - logger.info('@keyv/redis client reconnecting...'); + logger.debug('@keyv/redis client reconnecting...'); }); keyvRedisClient.on('disconnect', () => { - logger.warn('@keyv/redis client disconnected'); + logger.debug('@keyv/redis client disconnected'); }); // Start connection immediately and settle the gate created before client initialization. diff --git a/packages/api/src/cache/redisUtils.spec.ts b/packages/api/src/cache/redisUtils.spec.ts index 61df5d5710..e0927bd5b7 100644 --- a/packages/api/src/cache/redisUtils.spec.ts +++ b/packages/api/src/cache/redisUtils.spec.ts @@ -1,6 +1,7 @@ import IoRedis from 'ioredis'; import type { RedisClientType } from '@redis/client'; -import { duplicateIoRedisClient, scanKeys } from './redisUtils'; +import type { Redis, Cluster } from 'ioredis'; +import { createIoRedisSubscriber, duplicateIoRedisClient, scanKeys } from './redisUtils'; describe('duplicateIoRedisClient', () => { it('applies overrides to a single-node duplicate', () => { @@ -81,6 +82,40 @@ describe('duplicateIoRedisClient', () => { }); }); +describe.each([false, true])('createIoRedisSubscriber (cluster=%s)', (cluster) => { + it('retains subscriber commands without changing the publisher retry policy or options', () => { + const redisOptions = { + maxRetriesPerRequest: 0, + autoResendUnfulfilledCommands: false, + keyPrefix: 'subscriber-test::', + }; + const publisher = cluster + ? new IoRedis.Cluster([{ host: '127.0.0.1', port: 6379 }], { + lazyConnect: true, + redisOptions, + }) + : new IoRedis({ ...redisOptions, lazyConnect: true }); + const subscriber = createIoRedisSubscriber(publisher, 'test-subscriber'); + const options = subscriber.isCluster + ? (subscriber as Cluster).options.redisOptions + : (subscriber as Redis).options; + const publisherOptions = publisher.isCluster + ? (publisher as Cluster).options.redisOptions + : (publisher as Redis).options; + try { + expect(options).toMatchObject({ + maxRetriesPerRequest: null, + autoResendUnfulfilledCommands: true, + keyPrefix: redisOptions.keyPrefix, + }); + expect(publisherOptions).toMatchObject(redisOptions); + } finally { + subscriber.disconnect(); + publisher.disconnect(); + } + }); +}); + describe('scanKeys', () => { it('flattens node-redis v5 scan pages', async () => { const client = { diff --git a/packages/api/src/cache/redisUtils.ts b/packages/api/src/cache/redisUtils.ts index 91b23525ec..a4ce7572c1 100644 --- a/packages/api/src/cache/redisUtils.ts +++ b/packages/api/src/cache/redisUtils.ts @@ -53,19 +53,36 @@ export function duplicateIoRedisClient( * The heartbeat is the traffic the shared client gets for free. */ export function createIoRedisSubscriber(client: Redis | Cluster, label: string): Redis | Cluster { - const subscriber = duplicateIoRedisClient(client); + // Retain idempotent SUBSCRIBE/PING commands across lost acknowledgements. Publishers + // keep their fail-fast policy so uncertain application writes are never replayed. + const redisOptions: RedisOptions = { + maxRetriesPerRequest: null, + autoResendUnfulfilledCommands: true, + }; + const subscriber = duplicateIoRedisClient( + client, + client.isCluster + ? { redisOptions: { ...(client as Cluster).options.redisOptions, ...redisOptions } } + : redisOptions, + ); subscriber.on('error', (error: Error) => { logger.error(`${label} error:`, error); }); subscriber.on('ready', () => { logger.info(`${label} ready`); }); - startRedisHeartbeat({ + const stopHeartbeat = startRedisHeartbeat({ client: subscriber, intervalMs: cacheConfig.REDIS_SUBSCRIBER_PING_INTERVAL * 1000, timeoutMs: cacheConfig.REDIS_PING_TIMEOUT, label, }); + // An already-closed reconnecting socket need not emit another `end`. + const disconnect = subscriber.disconnect.bind(subscriber); + subscriber.disconnect = (reconnect?: boolean): void => { + if (reconnect !== true) stopHeartbeat(); + disconnect(reconnect); + }; return subscriber; } diff --git a/packages/api/src/cache/retry.spec.ts b/packages/api/src/cache/retry.spec.ts new file mode 100644 index 0000000000..9c1e2efc91 --- /dev/null +++ b/packages/api/src/cache/retry.spec.ts @@ -0,0 +1,66 @@ +import { createRedisRetryStrategy } from './retry'; + +describe('Redis reconnect policy', () => { + const log = jest.fn(); + const options = { + label: 'Redis', + maxAttempts: 0, + maxDelay: 3000, + initialDelay: 50, + log, + }; + + beforeEach(() => log.mockClear()); + + it('keeps retrying for the lifetime of the process without overflowing the backoff', () => { + const retry = createRedisRetryStrategy(options); + for (const attempt of [1, 10, 100, 1024, 100_000, Number.MAX_SAFE_INTEGER]) { + const delay = retry(attempt); + expect(Number.isFinite(delay)).toBe(true); + expect(delay).toBeGreaterThan(0); + expect(delay).toBeLessThanOrEqual(options.maxDelay); + } + }); + + it('retains jitter once the exponential delay has reached its cap', () => { + const random = jest.spyOn(Math, 'random'); + const retry = createRedisRetryStrategy(options); + random.mockReturnValueOnce(0).mockReturnValueOnce(0.999); + expect(retry(10_000)).toBe(3000); + expect(retry(10_000)).toBe(2001); + }); + + it.each([50, 100])( + 'grows the existing %sms backoff and respects a smaller cap', + (initialDelay) => { + jest.spyOn(Math, 'random').mockReturnValue(0); + const retry = createRedisRetryStrategy({ ...options, initialDelay }); + expect(retry(1)).toBe(initialDelay * 2); + expect(retry(2)).toBe(initialDelay * 4); + expect(createRedisRetryStrategy({ ...options, initialDelay, maxDelay: 1 })(100_000)).toBe(1); + }, + ); + + it.each([0, 1])('preserves finite overrides for a client counting from %s', (firstAttempt) => { + const retry = createRedisRetryStrategy({ ...options, maxAttempts: 2, firstAttempt }); + for (let attempt = firstAttempt; attempt <= 2; attempt++) { + expect(retry(attempt)).not.toBeNull(); + } + expect(retry(3)).toBeNull(); + expect(log).toHaveBeenLastCalledWith('Redis giving up after 2 reconnection attempts'); + }); + + it.each([0, 1])( + 'logs the first and every tenth failure for a client counting from %s', + (firstAttempt) => { + const retry = createRedisRetryStrategy({ ...options, firstAttempt }); + for (let attempt = firstAttempt; attempt < firstAttempt + 100; attempt++) { + retry(attempt); + } + expect(log).toHaveBeenCalledTimes(11); + expect(log.mock.calls[0][0]).toContain('attempt 1,'); + expect(log.mock.calls[1][0]).toContain('attempt 10,'); + expect(log.mock.calls[10][0]).toContain('attempt 100,'); + }, + ); +}); diff --git a/packages/api/src/cache/retry.ts b/packages/api/src/cache/retry.ts new file mode 100644 index 0000000000..aa4bc3dee8 --- /dev/null +++ b/packages/api/src/cache/retry.ts @@ -0,0 +1,36 @@ +type RedisRetryOptions = { + label: string; + maxAttempts: number; + maxDelay: number; + initialDelay: number; + firstAttempt?: number; + log: (message: string) => void; +}; + +/** Reconnect transport only; callers still own failed commands and their side effects. */ +export function createRedisRetryStrategy({ + label, + maxAttempts, + maxDelay, + initialDelay, + firstAttempt = 1, + log, +}: RedisRetryOptions): (attempt: number) => number | null { + const maxExponent = Math.max(0, Math.ceil(Math.log2(maxDelay / initialDelay))); + return (attempt) => { + if (maxAttempts > 0 && attempt > maxAttempts) { + log(`${label} giving up after ${maxAttempts} reconnection attempts`); + return null; + } + // Saturate the exponent before computing it, even after months of reconnects. + const exponent = Math.min(attempt, maxExponent); + const base = Math.min(initialDelay * 2 ** exponent, maxDelay); + // Keep jitter at the cap rather than clipping every retry to the same delay. + const delay = base - Math.floor(Math.random() * Math.min(base / 2, 1000)); + const failure = attempt - firstAttempt + 1; + if (failure === 1 || failure % 10 === 0) { + log(`${label} reconnecting... attempt ${failure}, delay ${delay}ms`); + } + return delay; + }; +} diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts index b579a54d1a..3bf962876a 100644 --- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts +++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts @@ -42,6 +42,210 @@ function deliverSequencedMessage( } describe('RedisEventTransport', () => { + it.each([ + { mode: 'ordinary', removal: 'unsubscribe' }, + { mode: 'frontier', removal: 'unsubscribe' }, + { mode: 'replay', removal: 'unsubscribe' }, + { mode: 'ordinary', removal: 'cleanup' }, + { mode: 'frontier', removal: 'cleanup' }, + { mode: 'replay', removal: 'cleanup' }, + ] as const)( + 'rejects $mode readiness first awaited after $removal and destroy', + async ({ mode, removal }) => { + const publisher = createMockPublisher(); + const subscriber = createMockSubscriber(); + subscriber.subscribe.mockImplementation(() => new Promise(() => undefined)); + const transport = new RedisEventTransport( + publisher as unknown as Redis, + subscriber as unknown as Redis, + ); + const options = + mode === 'replay' + ? { replay: { items: 10, bytes: 4096, ttlMs: 1000 } } + : { deferSequenceDelivery: true, captureSequenceFrontier: mode === 'frontier' }; + const attachment = transport.subscribe( + 'closed-before-await', + { onChunk: jest.fn() }, + options, + ); + + if (removal === 'unsubscribe') attachment.unsubscribe(); + else transport.cleanup('closed-before-await'); + transport.destroy(); + + await expect(attachment.ready).rejects.toThrow('Redis event transport is closed.'); + expect(publisher.eval).not.toHaveBeenCalled(); + expect(transport.getTrackedStreamIds()).toEqual([]); + }, + ); + + it.each(['abort', 'preempt'] as const)( + 'rejects a pending %s registration explicitly awaited after destroy', + async (kind) => { + const subscriber = createMockSubscriber(); + subscriber.subscribe.mockImplementation(() => new Promise(() => undefined)); + const transport = new RedisEventTransport( + createMockPublisher() as unknown as Redis, + subscriber as unknown as Redis, + ); + const registration = + kind === 'abort' + ? transport.onAbort('closed-control', jest.fn()) + : transport.onPreempt('closed-control', jest.fn()); + + transport.cleanup('closed-control'); + transport.destroy(); + + await expect(registration).rejects.toThrow('Redis event transport is closed.'); + expect(transport.getTrackedStreamIds()).toEqual([]); + }, + ); + + it('rejects both frontier readiness and synchronization when removed before their first await', async () => { + const publisher = createMockPublisher(); + const subscriber = createMockSubscriber(); + subscriber.subscribe.mockImplementation(() => new Promise(() => undefined)); + const transport = new RedisEventTransport( + publisher as unknown as Redis, + subscriber as unknown as Redis, + ); + const attachment = transport.subscribe( + 'closed-frontier-sync', + { onChunk: jest.fn() }, + { + deferSequenceDelivery: true, + captureSequenceFrontier: true, + }, + ); + const synchronized = attachment.syncReorderBuffer?.(); + + attachment.unsubscribe(); + transport.destroy(); + + await Promise.all([ + expect(attachment.ready).rejects.toThrow('Redis event transport is closed.'), + expect(synchronized).rejects.toThrow('Redis event transport is closed.'), + ]); + expect(publisher.eval).not.toHaveBeenCalled(); + }); + + it.each(['frontier', 'replay'] as const)( + 'preserves the original %s datastore readiness error', + async (mode) => { + const publisher = createMockPublisher(); + const failure = new Error('fixture EVAL failure'); + publisher.eval.mockRejectedValueOnce(failure); + const transport = new RedisEventTransport( + publisher as unknown as Redis, + createMockSubscriber() as unknown as Redis, + ); + const options = + mode === 'replay' + ? { replay: { items: 10, bytes: 4096, ttlMs: 1000 } } + : { deferSequenceDelivery: true, captureSequenceFrontier: true }; + const attachment = transport.subscribe('failed-readiness', { onChunk: jest.fn() }, options); + try { + await expect(attachment.ready).rejects.toBe(failure); + } finally { + attachment.unsubscribe(); + transport.destroy(); + } + }, + ); + + it('preserves a GET failure even when cleanup removes the stream before synchronization settles', async () => { + const publisher = createMockPublisher(); + let failRead!: (error: Error) => void; + publisher.get.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + failRead = reject; + }), + ); + const transport = new RedisEventTransport( + publisher as unknown as Redis, + createMockSubscriber() as unknown as Redis, + ); + const attachment = transport.subscribe('removed-sync', { onChunk: jest.fn() }); + await attachment.ready; + const synchronized = attachment.syncReorderBuffer?.(); + transport.cleanup('removed-sync'); + const failure = new Error('fixture GET failure'); + + failRead(failure); + + await expect(synchronized).rejects.toBe(failure); + transport.destroy(); + }); + + it('cancels a pending attachment locally and disconnects its subscriber only once', async () => { + jest.useFakeTimers(); + const publisher = createMockPublisher(); + const subscriber = { ...createMockSubscriber(), disconnect: jest.fn() }; + subscriber.subscribe.mockImplementation(() => new Promise(() => undefined)); + const transport = new RedisEventTransport( + publisher as unknown as Redis, + subscriber as unknown as Redis, + ); + try { + const attachment = transport.subscribe('unacknowledged-close', { onChunk: jest.fn() }); + const ready = attachment.ready?.then( + () => 'ready', + (error: unknown) => error, + ); + + transport.destroy(); + transport.destroy(); + + expect(await ready).toBeInstanceOf(Error); + expect(subscriber.disconnect).toHaveBeenCalledTimes(1); + expect(subscriber.unsubscribe).not.toHaveBeenCalled(); + jest.runAllTicks(); + expect(jest.getTimerCount()).toBe(0); + expect(() => transport.subscribe('after-close', { onChunk: jest.fn() })).toThrow( + 'Redis event transport is closed.', + ); + } finally { + transport.destroy(); + jest.useRealTimers(); + } + }); + + it('does not publish a frontier or reopen a channel after destruction during subscribe', async () => { + const publisher = createMockPublisher(); + const subscriber = createMockSubscriber(); + let acknowledge!: () => void; + subscriber.subscribe.mockImplementationOnce( + () => + new Promise((resolve) => { + acknowledge = resolve; + }), + ); + const transport = new RedisEventTransport( + publisher as unknown as Redis, + subscriber as unknown as Redis, + ); + const attachment = transport.subscribe( + 'destroyed-attachment', + { onChunk: jest.fn() }, + { + deferSequenceDelivery: true, + captureSequenceFrontier: true, + }, + ); + const ready = attachment.ready?.then( + () => 'ready', + (error: unknown) => error, + ); + + transport.destroy(); + acknowledge(); + + expect(await ready).toBeInstanceOf(Error); + expect(publisher.eval).not.toHaveBeenCalled(); + expect(transport.getTrackedStreamIds()).toEqual([]); + }); + it('delivers tagged events and preserves legacy in-memory callback arity', () => { const transport = new InMemoryEventTransport(); const onChunk = jest.fn(); @@ -2234,7 +2438,7 @@ describe('RedisEventTransport', () => { ); const streamId = 'reorder-abort-listener-reuse-test'; - transport.onAbort(streamId, () => {}); + const disposeAbort = await transport.onAbort(streamId, () => {}); const messageHandler = getMessageHandler(mockSubscriber); const channel = `stream:{${streamId}}:events`; @@ -2255,6 +2459,7 @@ describe('RedisEventTransport', () => { expect(secondRunChunks.map((chunk) => (chunk as { index: number }).index)).toEqual([0]); + disposeAbort(); transport.destroy(); }); }); diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts index d5aa8867b5..5b3845ec97 100644 --- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts @@ -1127,7 +1127,7 @@ describe('RedisEventTransport Integration Tests', () => { let abortReceived = false; // Register abort callback - transport.onAbort(streamId, () => { + const disposeAbort = await transport.onAbort(streamId, () => { abortReceived = true; }); @@ -1142,6 +1142,7 @@ describe('RedisEventTransport Integration Tests', () => { expect(abortReceived).toBe(true); + disposeAbort(); transport.destroy(); subscriber.disconnect(); }); @@ -1307,10 +1308,10 @@ describe('RedisEventTransport Integration Tests', () => { let callback2Called = false; // Multiple abort callbacks - transport.onAbort(streamId, () => { + const disposeFirst = await transport.onAbort(streamId, () => { callback1Called = true; }); - transport.onAbort(streamId, () => { + const disposeSecond = await transport.onAbort(streamId, () => { callback2Called = true; }); @@ -1323,6 +1324,8 @@ describe('RedisEventTransport Integration Tests', () => { expect(callback1Called).toBe(true); expect(callback2Called).toBe(true); + disposeFirst(); + disposeSecond(); transport.destroy(); subscriber.disconnect(); }); @@ -1341,7 +1344,7 @@ describe('RedisEventTransport Integration Tests', () => { const streamId = `abort-cleanup-${Date.now()}`; let abortReceived = false; - transport.onAbort(streamId, () => { + const disposeAbort = await transport.onAbort(streamId, () => { abortReceived = true; }); @@ -1349,6 +1352,7 @@ describe('RedisEventTransport Integration Tests', () => { // Cleanup the stream transport.cleanup(streamId); + disposeAbort(); // Emit abort after cleanup transport.emitAbort(streamId); diff --git a/packages/api/src/stream/__tests__/reconnect-reorder-desync.stream_integration.spec.ts b/packages/api/src/stream/__tests__/reconnect-reorder-desync.stream_integration.spec.ts index 3f4ad35742..ce98ca752a 100644 --- a/packages/api/src/stream/__tests__/reconnect-reorder-desync.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/reconnect-reorder-desync.stream_integration.spec.ts @@ -77,7 +77,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => { transport.destroy(); }); - test('abort callback survives across reconnect cycles', () => { + test('abort callback survives across reconnect cycles', async () => { const mockPublisher = createMockPublisher(); const mockSubscriber = { on: jest.fn(), @@ -94,7 +94,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => { let abortCallbackFired = false; // Register abort callback (simulates what createJob does) - transport.onAbort(streamId, () => { + const disposeAbort = await transport.onAbort(streamId, () => { abortCallbackFired = true; }); @@ -115,6 +115,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => { // Abort callback should fire — it was preserved across the reconnect expect(abortCallbackFired).toBe(true); + disposeAbort(); sub2.unsubscribe(); transport.destroy(); }); diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.ts index 05ea6fa894..e9f66cd150 100644 --- a/packages/api/src/stream/implementations/RedisEventTransport.ts +++ b/packages/api/src/stream/implementations/RedisEventTransport.ts @@ -251,6 +251,7 @@ interface SubscriptionFrontierWaiter { interface ChannelSubscriptionState { ready: Promise; + cancel?: () => void; phase: 'pending' | 'active'; /** A timed-out predecessor became active while this replacement was pending. */ fallbackActive: boolean; @@ -304,6 +305,12 @@ interface StreamSubscribers { * ``` */ export class RedisEventTransport implements IEventTransport { + private destroyed = false; + + private assertOpen(): void { + if (this.destroyed) throw new Error('Redis event transport is closed.'); + } + /** Redis client for publishing events */ private publisher: Redis | Cluster; /** Redis client for subscribing to events (separate connection required) */ @@ -312,6 +319,8 @@ export class RedisEventTransport implements IEventTransport { private streams = new Map(); /** Track channel subscription state: resolved promise = active, pending = in-flight */ private channelSubscriptions = new Map(); + /** Pending waits outlive channel removal until they settle or the transport closes. */ + private pendingSubscriptions = new Set(); /** Counter for generating unique subscriber IDs */ private subscriberIdCounter = 0; /** Coalescable chunk publications awaiting their window flush, per stream */ @@ -338,6 +347,7 @@ export class RedisEventTransport implements IEventTransport { } private getOrCreateStreamState(streamId: string): StreamSubscribers { + this.assertOpen(); const existing = this.streams.get(streamId); if (existing) { return existing; @@ -353,6 +363,7 @@ export class RedisEventTransport implements IEventTransport { streamId: string, replay = false, ): Promise { + this.assertOpen(); const subscriptionFrontierId = randomUUID(); let operationTimeout: ReturnType | undefined; const observed = new Promise((resolve, reject) => { @@ -396,6 +407,7 @@ export class RedisEventTransport implements IEventTransport { operationTimeout.unref?.(); }); const [raw] = await Promise.race([operation, operationDeadline]); + this.assertOpen(); const counter = replay ? (raw as [string, string[]])[0] : raw; const parsed = counter != null ? parseInt(String(counter), 10) : 0; const frontier = Number.isNaN(parsed) ? 0 : parsed; @@ -441,7 +453,7 @@ export class RedisEventTransport implements IEventTransport { // Set up message handler for all subscriptions this.subscriber.on('message', (channel: string, message: string) => { - this.handleMessage(channel, message); + if (!this.destroyed) this.handleMessage(channel, message); }); } @@ -677,6 +689,7 @@ export class RedisEventTransport implements IEventTransport { } private ensureChannelSubscription(streamId: string): Promise { + this.assertOpen(); const channel = CHANNELS.events(streamId); const existing = this.channelSubscriptions.get(channel); if (existing) { @@ -692,6 +705,10 @@ export class RedisEventTransport implements IEventTransport { fallbackActive: false, }; const deadline = new Promise((_, reject) => { + state.cancel = () => { + clearTimeout(timeout); + reject(new Error('Redis event transport is closed.')); + }; timeout = setTimeout(() => { expired = true; reject(new Error(`Timed out synchronizing Redis subscription for ${channel}`)); @@ -699,6 +716,7 @@ export class RedisEventTransport implements IEventTransport { timeout.unref?.(); }); const ready = Promise.race([operation, deadline]).then(() => { + this.assertOpen(); state.phase = 'active'; state.fallbackActive = false; logger.debug(`[RedisEventTransport] Subscription active for channel ${channel}`); @@ -707,7 +725,7 @@ export class RedisEventTransport implements IEventTransport { operation.then( () => { if (timeout != null) clearTimeout(timeout); - if (expired) { + if (expired && !this.destroyed) { const current = this.channelSubscriptions.get(channel); if (current == null || current === state) { if (current === state) this.channelSubscriptions.delete(channel); @@ -724,13 +742,19 @@ export class RedisEventTransport implements IEventTransport { }, ); this.channelSubscriptions.set(channel, state); - void ready.catch((err) => { - if (this.channelSubscriptions.get(channel) === state) { - this.channelSubscriptions.delete(channel); - if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel); - } - logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err); - }); + this.pendingSubscriptions.add(state); + void ready.then( + () => this.pendingSubscriptions.delete(state), + (err) => { + this.pendingSubscriptions.delete(state); + if (this.destroyed) return; + if (this.channelSubscriptions.get(channel) === state) { + this.channelSubscriptions.delete(channel); + if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel); + } + logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err); + }, + ); return ready; } @@ -1943,11 +1967,15 @@ export class RedisEventTransport implements IEventTransport { * Destroy all resources. */ destroy(): void { - for (const streamId of new Set( - [...this.subscriptionFrontierWaiters.values()].map((waiter) => waiter.streamId), - )) { - this.releaseSubscriptionFrontiers(streamId); + if (this.destroyed) return; + this.destroyed = true; + for (const waiter of this.subscriptionFrontierWaiters.values()) { + clearTimeout(waiter.timeout); + waiter.reject(new Error('Redis event transport is closed.')); } + this.subscriptionFrontierWaiters.clear(); + for (const state of this.pendingSubscriptions) state.cancel?.(); + this.pendingSubscriptions.clear(); for (const streamId of this.pendingBatches.keys()) { this.discardCoalescedChunks(streamId); } @@ -1969,10 +1997,6 @@ export class RedisEventTransport implements IEventTransport { state.abortAckWaiters.clear(); } - for (const channel of this.channelSubscriptions.keys()) { - this.subscriber.unsubscribe(channel).catch(() => {}); - } - this.channelSubscriptions.clear(); this.streams.clear();