🐦‍🔥 fix: Recover Redis Clients After Extended Outages (#18164)

* 🐦‍🔥 fix: Recover Redis Clients After Extended Outages

* fix: Retain Subscriber Registrations Across Repeated Disconnects

* test: Diagnose Redis Fixture Startup Failures

* fix: Close Redis Invalidation Connections During Outages

* fix: Bound Redis Subscriber Lifetimes During Shutdown

* fix: Retain Pending Redis Subscription Cancellation After Removal

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-11-16 19:04:57 -05:00 • committed by GitHub
parent 687af9ac5a
commit e1fc59c9cf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 1586 additions and 113 deletions

View file

@ -0,0 +1,218 @@
import Redis from 'ioredis';
import http from 'node:http';
import type { SubagentTaskControlHandler } from './subagentTaskRouting';
import type { RespServer } from '~/cache/__tests__/resp.helper';
import {
RedisSubagentTaskControlTransport,
SubagentTaskOwnerUnavailableError,
} from './subagentTaskRouting';
import {
setupGracefulShutdown,
registerShutdownTask,
__resetShutdownStateForTests,
} from '~/app/shutdown';
import { createIoRedisSubscriber, duplicateIoRedisClient } from '~/cache/redisUtils';
import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper';
jest.mock('@librechat/data-schemas', () => ({
logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() },
}));
jest.mock('~/utils', () => ({
...jest.requireActual('~/utils/math'),
...jest.requireActual('~/utils/common'),
}));
const handler: SubagentTaskControlHandler = {
claim: () => ({ status: 'not_found' }),
control: () => ({ status: 'not_found' }),
list: () => [],
retainsTaskOwnership: () => false,
cancelScope: () => 0,
};
const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGHUP'];
describe('Redis task-routing lifecycle', () => {
let server: RespServer;
let shared: Redis;
let publisher: Redis;
let subscriber: Redis;
let transport: RedisSubagentTaskControlTransport;
let shutdownExit: jest.SpyInstance | undefined;
let originalListeners: Map<NodeJS.Signals, NodeJS.SignalsListener[]>;
beforeEach(async () => {
originalListeners = new Map(signals.map((signal) => [signal, process.listeners(signal)]));
__resetShutdownStateForTests();
shutdownExit = undefined;
server = await startRespServer();
shared = new Redis(server.url, {
maxRetriesPerRequest: 0,
autoResendUnfulfilledCommands: false,
retryStrategy: () => 20,
});
shared.on('error', () => undefined);
await waitFor(() => shared.status === 'ready');
publisher = duplicateIoRedisClient(shared, { enableOfflineQueue: false }) as Redis;
publisher.on('error', () => undefined);
subscriber = createIoRedisSubscriber(shared, 'task-lifecycle-subscriber') as Redis;
transport = new RedisSubagentTaskControlTransport(publisher, subscriber);
});
afterEach(async () => {
try {
await transport.destroy();
subscriber.disconnect();
publisher.disconnect();
shared.disconnect();
await server.close();
await new Promise((resolve) => setImmediate(resolve));
if (shutdownExit) await waitFor(() => shutdownExit!.mock.calls.length === 1, 500);
} finally {
__resetShutdownStateForTests();
for (const signal of signals) {
for (const listener of process.listeners(signal)) {
if (!originalListeners.get(signal)?.includes(listener)) {
process.removeListener(signal, listener);
}
}
}
shutdownExit?.mockRestore();
}
});
it('cancels a readiness wait when destroyed without closing its borrowed publisher', async () => {
await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready');
await server.close();
await waitFor(
() => publisher.status === 'reconnecting' && subscriber.status === 'reconnecting',
);
const publisherDisconnect = jest.spyOn(publisher, 'disconnect');
const publisherOnce = jest.spyOn(publisher, 'once');
const subscriberOnce = jest.spyOn(subscriber, 'once');
let result: unknown;
void transport.bind(handler).then(
() => {
result = 'bound';
},
(error: unknown) => {
result = error;
},
);
const readinessListeners = [publisherOnce, subscriberOnce].map((spy) =>
spy.mock.calls.filter(([event]) => event === 'ready' || event === 'end'),
);
let destroyed = false;
void transport.destroy().then(() => {
destroyed = true;
});
await waitFor(() => result != null && destroyed, 500);
expect(result).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
for (const [index, client] of [publisher, subscriber].entries()) {
for (const [event, listener] of readinessListeners[index]) {
expect(client.listeners(event)).not.toContain(listener);
}
}
expect(publisherDisconnect).not.toHaveBeenCalled();
});
it('does not subscribe when readiness resolves just before destruction', async () => {
await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready');
const subscribe = jest.spyOn(subscriber, 'subscribe');
const binding = transport.bind(handler).then(
() => 'bound',
(error: unknown) => error,
);
await transport.destroy();
expect(await binding).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
expect(subscribe).not.toHaveBeenCalled();
expect(subscriber.listenerCount('message')).toBe(0);
await expect(publisher.ping()).resolves.toBe('PONG');
});
it('cancels an unacknowledged initial subscription on destruction', async () => {
await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready');
server.silent = true;
let result: unknown;
void transport.bind(handler).then(
() => {
result = 'bound';
},
(error: unknown) => {
result = error;
},
);
await waitFor(() => server.commands.some((frame) => frame[0] === 'subscribe'));
let destroyed = false;
void transport.destroy().then(() => {
destroyed = true;
});
await waitFor(() => result != null && destroyed, 500);
expect(result).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
expect(subscriber.listenerCount('message')).toBe(0);
});
it('stops its heartbeat on intentional disconnect even when the closed socket emits no end', async () => {
await transport.bind(handler);
await server.close();
await waitFor(() => subscriber.status === 'reconnecting');
const listeners = subscriber.listenerCount('end');
expect(listeners).toBeGreaterThan(0);
subscriber.disconnect(true);
expect(subscriber.listenerCount('end')).toBe(listeners);
subscriber.disconnect();
expect(subscriber.listenerCount('end')).toBe(listeners - 1);
});
it('allows later shutdown tasks to run while Redis is unreachable', async () => {
await transport.bind(handler);
await server.close();
await waitFor(() => subscriber.status === 'reconnecting');
const steps: string[] = [];
const exit = jest.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
shutdownExit = exit;
registerShutdownTask(
'generation job manager',
() => {
steps.push('generation');
},
{ priority: 100 },
);
registerShutdownTask(
'subagent task store',
async () => {
await transport.destroy();
publisher.disconnect();
steps.push('task-routing');
},
{ priority: 90 },
);
registerShutdownTask(
'favorites persistence',
() => {
steps.push('favorites');
},
{ priority: -100 },
);
setupGracefulShutdown(http.createServer());
const listener = process
.listeners('SIGTERM')
.find(
(entry) => !originalListeners.get('SIGTERM')?.includes(entry),
) as NodeJS.SignalsListener;
listener('SIGTERM');
await waitFor(() => exit.mock.calls.length === 1, 500);
expect(steps).toEqual(['generation', 'task-routing', 'favorites']);
expect(exit).toHaveBeenCalledWith(0);
await transport.destroy();
});
});

View file

@ -184,6 +184,144 @@ function taskHandler(
}
describe('RedisSubagentTaskControlTransport', () => {
it('bounds a missing initial SUBSCRIBE acknowledgement and closes only its subscriber', async () => {
jest.useFakeTimers();
const bus = new FakeRedisBus();
const publisher = bus.createClient();
const subscriber = bus.createClient();
jest.spyOn(subscriber, 'subscribe').mockImplementation(() => new Promise(() => undefined));
const transport = new RedisSubagentTaskControlTransport(
asRedis(publisher),
asRedis(subscriber),
);
try {
const binding = transport.bind(taskHandler()).then(
() => 'bound',
(error: unknown) => error,
);
await jest.advanceTimersByTimeAsync(10_000);
expect(await binding).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
expect(subscriber.disconnected).toBe(true);
expect(subscriber.listeners.size).toBe(0);
expect(publisher.disconnected).toBe(false);
expect(jest.getTimerCount()).toBe(0);
} finally {
await transport.destroy();
jest.useRealTimers();
}
});
it('ignores a message callback queued before destruction', async () => {
const bus = new FakeRedisBus();
const subscriber = bus.createClient();
const transport = new RedisSubagentTaskControlTransport(
asRedis(bus.createClient()),
asRedis(subscriber),
);
const control = jest.fn(() => ({ status: 'not_found' as const }));
await transport.bind(taskHandler({ control }));
const listener = [...subscriber.listeners][0];
const channel = [...subscriber.channels][0];
await transport.destroy();
listener(
channel,
JSON.stringify({
version: 1,
kind: 'request',
operation: 'control',
requestId: 'late-request',
requesterId: 'requester',
scopeId: 'scope-1',
taskId: 'task-1',
invocationId: 'invocation-1',
expiresAt: Date.now() + 1000,
command: { action: 'queue', message: 'late control' },
}),
);
await Promise.resolve();
expect(control).not.toHaveBeenCalled();
});
it('does not publish a control reply that settles after destruction', async () => {
const bus = new FakeRedisBus();
const publisher = bus.createClient();
const subscriber = bus.createClient();
const transport = new RedisSubagentTaskControlTransport(
asRedis(publisher),
asRedis(subscriber),
);
let finish!: (value: SubagentTaskControlResult) => void;
const control = jest.fn(
() =>
new Promise<SubagentTaskControlResult>((resolve) => {
finish = resolve;
}),
);
await transport.bind(taskHandler({ control }));
const channel = [...subscriber.channels][0];
bus.publish(
channel,
JSON.stringify({
version: 1,
kind: 'request',
operation: 'control',
requestId: 'pending-request',
requesterId: 'requester',
scopeId: 'scope-1',
taskId: 'task-1',
invocationId: 'invocation-1',
expiresAt: Date.now() + 1000,
command: { action: 'queue', message: 'pending control' },
}),
);
await Promise.resolve();
expect(control).toHaveBeenCalledTimes(1);
await transport.destroy();
const publish = jest.spyOn(publisher, 'publish');
finish({ status: 'not_found' });
await Promise.resolve();
await Promise.resolve();
expect(publish).not.toHaveBeenCalled();
});
it('does not issue queued owner refreshes after destruction', async () => {
jest.useFakeTimers();
const bus = new FakeRedisBus();
const publisher = bus.createClient();
const transport = new RedisSubagentTaskControlTransport(
asRedis(publisher),
asRedis(bus.createClient()),
{ registrationHeartbeatMs: 1 },
);
const tasks = Array.from({ length: 36 }, (_, index) => snapshot({ taskId: `task-${index}` }));
const releases: Array<() => void> = [];
try {
await transport.bind(taskHandler({ list: () => tasks }));
for (const task of tasks) await transport.registerTask('scope-1', task.taskId, 60_000);
bus.registrationHook = () =>
new Promise<void>((resolve) => {
releases.push(resolve);
});
const evalCommand = jest.spyOn(publisher, 'eval');
await jest.advanceTimersByTimeAsync(1);
expect(releases).toHaveLength(32);
await transport.destroy();
for (const release of releases) release();
await jest.advanceTimersByTimeAsync(1);
expect(evalCommand).toHaveBeenCalledTimes(32);
expect(jest.getTimerCount()).toBe(0);
} finally {
await transport.destroy();
jest.useRealTimers();
}
});
it('waits for the fail-fast publisher before reporting itself bound', async () => {
const bus = new FakeRedisBus();
const publisher = new EventEmitter() as EventEmitter & { status: string };

View file

@ -355,7 +355,13 @@ function boundedControlResult(result: SubagentTaskControlResult): SubagentTaskCo
};
}
async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
async function waitForRedisConnectionReady(
client: RedisClient,
signal: AbortSignal,
): Promise<void> {
if (signal.aborted) {
throw new SubagentTaskOwnerUnavailableError();
}
if (client.status == null || client.status === 'ready') {
return;
}
@ -367,6 +373,7 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
clearTimeout(timeout);
client.off('ready', onReady);
client.off('end', onEnd);
signal.removeEventListener('abort', onEnd);
};
const onReady = () => {
cleanup();
@ -381,11 +388,14 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
reject(new SubagentTaskOwnerUnavailableError());
}, DEFAULT_READY_TIMEOUT_MS);
timeout.unref?.();
signal.addEventListener('abort', onEnd, { once: true });
client.once('ready', onReady);
client.once('end', onEnd);
/** Close the status-check/listener-registration race: ioredis may become ready
* synchronously between the check above and installing these listeners. */
if (client.status === 'ready') {
if (signal.aborted) {
onEnd();
} else if (client.status === 'ready') {
onReady();
} else if (client.status === 'end') {
onEnd();
@ -397,9 +407,10 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
async function waitForRedisReady(
client: RedisClient,
signal: AbortSignal,
options: { eagerClusterMasters?: boolean } = {},
): Promise<void> {
await waitForRedisConnectionReady(client);
await waitForRedisConnectionReady(client, signal);
if (options.eagerClusterMasters !== true || !client.isCluster) {
return;
}
@ -407,10 +418,41 @@ async function waitForRedisReady(
* fail-fast publisher cannot admit requests until every possible write target is
* connected; otherwise the first command to a cold shard would be rejected. */
await Promise.all(
(client as Cluster).nodes('master').map((node) => waitForRedisConnectionReady(node)),
(client as Cluster).nodes('master').map((node) => waitForRedisConnectionReady(node, signal)),
);
}
/** Bounds the initial SUBSCRIBE acknowledgement and cancels it with its owner. */
async function waitForRedisSubscription(
operation: Promise<unknown>,
signal: AbortSignal,
): Promise<void> {
await new Promise<void>((resolve, reject) => {
const cleanup = (): void => {
clearTimeout(timeout);
signal.removeEventListener('abort', onAbort);
};
const onAbort = (): void => {
cleanup();
reject(new SubagentTaskOwnerUnavailableError());
};
const timeout = setTimeout(onAbort, DEFAULT_READY_TIMEOUT_MS);
timeout.unref?.();
signal.addEventListener('abort', onAbort, { once: true });
operation.then(
() => {
cleanup();
resolve();
},
(error: unknown) => {
cleanup();
reject(error);
},
);
if (signal.aborted) onAbort();
});
}
function isSnapshot(value: unknown): value is SubagentTaskSnapshot {
if (value == null || typeof value !== 'object' || Array.isArray(value)) {
return false;
@ -734,7 +776,9 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
private registrationHeartbeat?: ReturnType<typeof setInterval>;
private registrationRefresh?: Promise<void>;
private destroyed = false;
private readonly lifecycle = new AbortController();
/** Borrows the publisher; owns the dedicated subscriber and its pending waits. */
constructor(
private readonly publisher: RedisClient,
private readonly subscriber: RedisClient,
@ -764,13 +808,22 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
/** The publisher fails fast instead of queueing commands, so opening HTTP
* admission before it is ready would turn healthy startup lag into false
* `unavailable` results. Both dedicated connections are part of readiness. */
await Promise.all([
waitForRedisReady(this.publisher, { eagerClusterMasters: true }),
waitForRedisReady(this.subscriber),
]);
this.subscriber.on('message', this.onMessage);
this.ready = this.subscriber.subscribe(this.channel(this.instanceId)).then(() => undefined);
await this.ready;
try {
await Promise.all([
waitForRedisReady(this.publisher, this.lifecycle.signal, { eagerClusterMasters: true }),
waitForRedisReady(this.subscriber, this.lifecycle.signal),
]);
if (this.destroyed) throw new SubagentTaskOwnerUnavailableError();
this.subscriber.on('message', this.onMessage);
this.ready = waitForRedisSubscription(
this.subscriber.subscribe(this.channel(this.instanceId)),
this.lifecycle.signal,
);
await this.requireReady();
} catch (error) {
await this.destroy();
throw error;
}
}
async registerTask(scopeId: string, taskId: string, ttlMs: number): Promise<void> {
@ -963,6 +1016,8 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
return;
}
this.destroyed = true;
this.lifecycle.abort();
this.handler = undefined;
for (const pending of this.pending.values()) {
clearTimeout(pending.retry);
clearTimeout(pending.timeout);
@ -979,12 +1034,15 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
this.registrationHeartbeat = undefined;
}
this.subscriber.off('message', this.onMessage);
await this.subscriber.unsubscribe(this.channel(this.instanceId)).catch(() => undefined);
this.subscriber.disconnect();
}
private readonly onMessage = (channel: string, message: string): void => {
if (channel !== this.channel(this.instanceId) || message.length > MAX_ROUTED_MESSAGE_CHARS) {
if (
this.destroyed ||
channel !== this.channel(this.instanceId) ||
message.length > MAX_ROUTED_MESSAGE_CHARS
) {
return;
}
let parsed: unknown;
@ -1027,6 +1085,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
}
private async handleRequest(request: RoutedRequest): Promise<void> {
if (this.destroyed) return;
if (Date.now() > request.expiresAt + REQUEST_CLOCK_SKEW_MS) {
/** The caller stopped waiting for this long ago and has been told it was
* unavailable, so applying it now would steer a child it believes untouched. */
@ -1094,6 +1153,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
),
);
}
if (this.destroyed) return;
const serializedResult = JSON.stringify(result);
/** Retaining the result rather than the envelope lets a later caller retry,
* which carries its own correlation id, recover a response it never received. */
@ -1102,6 +1162,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
}
serialized = successResponse(request.requestId, serializedResult);
} catch (error) {
if (this.destroyed) return;
logger.error('[subagentTaskRouting] Owner failed to process a routed command', error);
serialized = failureResponse(request.requestId);
}
@ -1306,6 +1367,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
}
private async publish(channel: string, value: string): Promise<number> {
if (this.destroyed) throw new SubagentTaskOwnerUnavailableError();
const delivered = await this.publisher.publish(channel, value);
return typeof delivered === 'number' ? delivered : 0;
}
@ -1451,5 +1513,6 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
throw new SubagentTaskOwnerUnavailableError();
}
await this.ready;
if (this.destroyed) throw new SubagentTaskOwnerUnavailableError();
}
}

View file

@ -25,10 +25,8 @@ describe('createRedisConfigInvalidationPubSub', () => {
});
afterEach(async () => {
for (const { invalidation, pubsub } of opened) {
for (const { invalidation } of opened) {
await invalidation.close().catch(() => undefined);
(pubsub.publisher as Redis).disconnect();
(pubsub.subscriber as Redis).disconnect();
}
for (const client of clients) {
client.disconnect();
@ -196,8 +194,7 @@ describe('createRedisConfigInvalidationPubSub', () => {
server = await startRespServer();
});
it('subscribes once Redis comes back when the first SUBSCRIBE failed during an outage', async () => {
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
it('subscribes once Redis comes back when the first SUBSCRIBE is pending during an outage', async () => {
const channel = 'librechat:config:invalidate:deploy-a';
const client = await connect();
const { port } = server;
@ -205,13 +202,9 @@ describe('createRedisConfigInvalidationPubSub', () => {
await waitFor(() => client.status !== 'ready');
const late = await startInstanceOn(client, 'deploy-a');
await waitFor(
() =>
error.mock.calls.some(([message]) =>
String(message).includes('Subscribing to cross-instance invalidations failed'),
),
5000,
);
const subscribe = jest.spyOn(late.pubsub.subscriber, 'subscribe');
await waitFor(() => (late.pubsub.subscriber as Redis).status === 'reconnecting');
expect(server.commands.filter((frame) => frame[0]?.toUpperCase() === 'SUBSCRIBE')).toEqual([]);
server = await startRespServer(port);
await waitFor(subscribed(channel, 1), 5000);
@ -222,6 +215,48 @@ describe('createRedisConfigInvalidationPubSub', () => {
await waitFor(() => late.clearOverrides.mock.calls.length === 1);
expect(late.clearOverrides).toHaveBeenCalledWith('tenant-a');
expect(subscribe).not.toHaveBeenCalled();
});
it.each([false, true])(
'closes its owned connections during an outage (subscription acknowledged=%s)',
async (acknowledged) => {
const client = await connect();
const early = acknowledged ? await startInstanceOn(client, 'deploy-a') : undefined;
if (early) {
await waitFor(() => (early.pubsub.subscriber as Redis).mode === 'subscriber');
}
await server.close();
await waitFor(() => client.status !== 'ready');
const instance = early ?? (await startInstanceOn(client, 'deploy-a'));
const subscriber = instance.pubsub.subscriber as Redis;
const publisher = instance.pubsub.publisher as Redis;
await waitFor(() => subscriber.status === 'reconnecting');
const disconnectSubscriber = jest.spyOn(subscriber, 'disconnect');
const disconnectPublisher = jest.spyOn(publisher, 'disconnect');
const unsubscribe = jest.spyOn(subscriber, 'unsubscribe');
const started = Date.now();
await instance.invalidation.close();
await instance.invalidation.close();
expect(Date.now() - started).toBeLessThan(1000);
expect(disconnectSubscriber).toHaveBeenCalledTimes(1);
expect(disconnectPublisher).toHaveBeenCalledTimes(1);
expect(unsubscribe).not.toHaveBeenCalled();
},
);
it('leaves the shared client usable when the owned pair is closed', async () => {
const client = await connect();
const instance = await startInstanceOn(client, 'deploy-a');
await waitFor(() => (instance.pubsub.subscriber as Redis).mode === 'subscriber');
await instance.invalidation.close();
await expect(client.ping()).resolves.toBe('PONG');
await waitFor(() => (instance.pubsub.subscriber as Redis).status === 'end');
await waitFor(() => (instance.pubsub.publisher as Redis).status === 'end');
});
it('returns no pub/sub without Redis', async () => {

View file

@ -183,6 +183,75 @@ describe('createConfigCacheInvalidation', () => {
expect(second.clearOverrides).not.toHaveBeenCalled();
});
it('releases borrowed listeners without waiting for an unavailable unsubscribe acknowledgement', async () => {
const broker = createBroker();
const subscriber = broker.subscriber();
jest.spyOn(subscriber, 'unsubscribe').mockImplementation(() => new Promise(() => undefined));
const invalidation = createConfigCacheInvalidation({
clears: [],
pubsub: { publisher: broker.publisher(), subscriber },
});
let closed = false;
void invalidation.close().then(() => {
closed = true;
});
await flush();
expect(closed).toBe(true);
expect(subscriber.events.listenerCount('message')).toBe(0);
expect(subscriber.events.listenerCount('ready')).toBe(0);
});
it('releases an owned pub/sub pair once instead of awaiting unsubscribe', async () => {
const broker = createBroker();
const subscriber = broker.subscriber();
const unsubscribe = jest.spyOn(subscriber, 'unsubscribe');
const close = jest.fn();
const invalidation = createConfigCacheInvalidation({
clears: [],
pubsub: { publisher: broker.publisher(), subscriber, close },
});
await flush();
await invalidation.close();
await invalidation.close();
subscriber.events.emit('ready');
await flush();
expect(close).toHaveBeenCalledTimes(1);
expect(unsubscribe).not.toHaveBeenCalled();
expect(subscriber.subscribe).toHaveBeenCalledTimes(1);
expect(subscriber.events.listenerCount('ready')).toBe(0);
expect(subscriber.events.listenerCount('message')).toBe(0);
});
it('does not report a pending subscription rejected by intentional close as an outage', async () => {
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
const broker = createBroker();
const subscriber = broker.subscriber();
let rejectSubscribe: (error: Error) => void = () => undefined;
subscriber.subscribe.mockImplementation(
() =>
new Promise((_resolve, reject) => {
rejectSubscribe = reject;
}),
);
const invalidation = createConfigCacheInvalidation({
clears: [],
pubsub: {
publisher: broker.publisher(),
subscriber,
close: () => rejectSubscribe(new Error('Connection is closed.')),
},
});
await invalidation.close();
await flush();
expect(error).not.toHaveBeenCalled();
});
it('subscribes again when the subscriber becomes ready after a failed SUBSCRIBE', async () => {
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
const broker = createBroker();

View file

@ -31,6 +31,8 @@ export interface ConfigInvalidationPubSub {
publisher: ConfigInvalidationPublisher;
subscriber: ConfigInvalidationSubscriber;
channel?: string;
/** Releases owned connections without waiting for a Redis acknowledgement. */
close?(): void;
}
/** One cache a config write invalidates; `tenantId` scopes it when the cache supports that. */
@ -158,6 +160,9 @@ export function createConfigCacheInvalidation(params: {
},
(error: unknown) => {
subscribing = false;
if (closed) {
return;
}
logger.error(
'[invalidateConfigCaches] Subscribing to cross-instance invalidations failed; retrying when the subscriber reconnects. Until then this instance may serve cached config until it expires:',
getSafeErrorMetadata(error),
@ -209,13 +214,17 @@ export function createConfigCacheInvalidation(params: {
}
async function close(): Promise<void> {
if (!pubsub) {
if (!pubsub || closed) {
return;
}
closed = true;
pubsub.subscriber.off('ready', subscribe);
pubsub.subscriber.off('message', onMessage);
await pubsub.subscriber.unsubscribe(channel);
if (pubsub.close) {
pubsub.close();
return;
}
void pubsub.subscriber.unsubscribe(channel).catch(() => undefined);
}
return { invalidate, reload, close };
@ -242,9 +251,14 @@ export function createRedisConfigInvalidationPubSub(params: {
publisher.on('error', (error: Error) => {
logger.error('[ConfigInvalidation] publisher error:', getSafeErrorMetadata(error));
});
const subscriber = createIoRedisSubscriber(client, '[ConfigInvalidation] subscriber');
return {
publisher,
subscriber: createIoRedisSubscriber(client, '[ConfigInvalidation] subscriber'),
subscriber,
channel: `${CONFIG_INVALIDATION_CHANNEL}:${namespace?.trim() || 'default'}`,
close: () => {
subscriber.disconnect();
publisher.disconnect();
},
};
}

View file

@ -0,0 +1,217 @@
import type { Redis, Cluster } from 'ioredis';
import type { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';
import type { RedisClientsModule } from './redisClients.helper';
import { startRedis, sleep, waitUntil } from './outage.helper';
import { isRedisReady } from '~/app/readiness';
jest.mock('@librechat/data-schemas', () => ({
logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
}));
jest.mock('~/utils', () => ({
...jest.requireActual('~/utils/math'),
...jest.requireActual('~/utils/common'),
}));
describe.each([false, true])('Redis outage recovery (cluster=%s)', (cluster) => {
let server: Awaited<ReturnType<typeof startRedis>> | undefined;
let clients: RedisClientsModule | undefined;
let subscriber: Redis | Cluster | undefined;
let env: NodeJS.ProcessEnv;
const transports: RedisEventTransport[] = [];
const extraClients: (Redis | Cluster)[] = [];
const ready = (): boolean =>
isRedisReady({
useRedis: true,
ioredis: clients?.ioredisClient ?? null,
keyv: clients?.keyvRedisClient ?? null,
});
const connect = async (): Promise<void> => {
clients = await import('../redisClients');
await clients.waitForKeyvRedisClient();
await waitUntil(ready);
};
beforeEach(async () => {
env = { ...process.env };
server = await startRedis(cluster);
Object.assign(process.env, {
USE_REDIS: 'true',
USE_REDIS_CLUSTER: String(cluster),
REDIS_URI: server.url,
REDIS_RETRY_MAX_DELAY: '100',
REDIS_PING_INTERVAL: '0',
REDIS_SUBSCRIBER_PING_INTERVAL: '0',
REDIS_KEY_PREFIX: '',
REDIS_KEY_PREFIX_VAR: '',
REDIS_USERNAME: '',
REDIS_PASSWORD: '',
REDIS_CA: '',
REDIS_ENABLE_OFFLINE_QUEUE: 'true',
});
delete process.env.REDIS_RETRY_MAX_ATTEMPTS;
jest.resetModules();
});
afterEach(async () => {
for (const transport of transports.splice(0)) {
transport.destroy();
}
for (const client of extraClients.splice(0)) {
client.disconnect();
}
subscriber?.disconnect();
subscriber = undefined;
clients?.ioredisClient?.disconnect();
const keyv = clients?.keyvRedisClient;
try {
if (keyv?.isOpen) {
keyv.destroy();
}
} catch (error) {
// Cluster teardown can encounter a master that exhausted its finite budget.
if (!(error instanceof Error) || error.message !== 'The client is closed') {
throw error;
}
} finally {
clients = undefined;
try {
await server?.close();
} finally {
server = undefined;
process.env = env;
}
}
});
it('recovers both clients and subscriptions after 20 seconds without replaying an uncertain write', async () => {
await connect();
const io = clients!.ioredisClient!;
const keyv = clients!.keyvRedisClient!;
const { createIoRedisSubscriber } = await import('../redisUtils');
subscriber = createIoRedisSubscriber(io, 'outage-subscriber');
const messages: string[] = [];
subscriber.on('message', (_channel: string, value: string) => messages.push(value));
await waitUntil(() => subscriber?.status === 'ready');
await subscriber.subscribe('outage-events');
await io.publish('outage-events', 'before');
await waitUntil(() => messages.includes('before'));
server!.dropReplies();
const pendingWrite = io.incr('outage-counter').then(
() => 'resolved',
() => 'rejected',
);
const pendingKeyvWrite = keyv.incr('keyv-outage-counter').then(
() => 'resolved',
() => 'rejected',
);
let applied = false;
const deadline = Date.now() + 5000;
while (!applied && Date.now() < deadline) {
applied =
(await server!.admin.get('::outage-counter')) === '1' &&
(await server!.admin.get('keyv-outage-counter')) === '1';
if (!applied) await sleep(20);
}
expect(applied).toBe(true);
server!.block();
await waitUntil(() => !ready());
await sleep(20_000);
expect(ready()).toBe(false);
await expect(pendingWrite).resolves.toBe('rejected');
await expect(pendingKeyvWrite).resolves.toBe('rejected');
server!.restore();
await waitUntil(ready);
await waitUntil(() => subscriber?.status === 'ready');
await expect(io.set('after-outage', 'ioredis')).resolves.toBe('OK');
await expect(io.get('after-outage')).resolves.toBe('ioredis');
await expect(keyv.set('keyv-after-outage', 'keyv')).resolves.toBe('OK');
await expect(keyv.get('keyv-after-outage')).resolves.toBe('keyv');
await expect(server!.admin.get('::outage-counter')).resolves.toBe('1');
await expect(server!.admin.get('keyv-outage-counter')).resolves.toBe('1');
await io.publish('outage-events', 'after');
await waitUntil(() => messages.includes('after'));
expect(messages).toEqual(['before', 'after']);
}, 40_000);
it('keeps stream and abort subscriptions after an interrupted resubscribe acknowledgement', async () => {
await connect();
const { createIoRedisSubscriber, duplicateIoRedisClient } = await import('../redisUtils');
const { RedisEventTransport } = await import('~/stream/implementations/RedisEventTransport');
const io = clients!.ioredisClient!;
subscriber = createIoRedisSubscriber(io, 'outage-owner');
let readyEvents = 0;
subscriber.on('ready', () => readyEvents++);
const publisher = duplicateIoRedisClient(io);
extraClients.push(publisher);
const owner = new RedisEventTransport(io, subscriber);
const peer = new RedisEventTransport(publisher, createIoRedisSubscriber(io, 'outage-peer'));
transports.push(owner, peer);
const streamId = 'interrupted-subscription';
const channel = `stream:{${streamId}}:events`;
const chunks: unknown[] = [];
let abortedGeneration: number | undefined;
const attachment = owner.subscribe(streamId, { onChunk: (event) => chunks.push(event) });
await attachment.ready;
const unsubscribeAbort = await owner.onAbort(streamId, (generation) => {
abortedGeneration = generation;
});
await waitUntil(() => publisher.status === 'ready');
await peer.emitChunk(streamId, { value: 'before' });
await waitUntil(() => chunks.length === 1);
server!.block();
await waitUntil(() => !ready());
server!.interruptNextSubscription();
server!.restore();
await waitUntil(() => server!.interruptedSubscriptions === 1);
await waitUntil(
() => ready() && subscriber?.status === 'ready' && readyEvents >= (cluster ? 2 : 3),
);
const isSubscribed = async (): Promise<boolean> => {
const counts = await server!.admin.pubsub('NUMSUB', channel);
return Array.isArray(counts) && counts[1] === 1;
};
let subscribed = await isSubscribed();
const deadline = Date.now() + 2000;
while (!subscribed && Date.now() < deadline) {
await sleep(20);
subscribed = await isSubscribed();
}
expect(subscribed).toBe(true);
await peer.emitChunk(streamId, { value: 'after' });
peer.emitAbort(streamId, 7);
await waitUntil(() => chunks.length === 2 && abortedGeneration === 7);
expect(chunks).toEqual([{ value: 'before' }, { value: 'after' }]);
unsubscribeAbort();
attachment.unsubscribe();
}, 15_000);
it('honors an explicit finite reconnect budget and remains not ready', async () => {
process.env.REDIS_RETRY_MAX_ATTEMPTS = '2';
await connect();
const { createIoRedisSubscriber } = await import('../redisUtils');
subscriber = createIoRedisSubscriber(clients!.ioredisClient!, 'finite-subscriber');
await waitUntil(() => subscriber?.status === 'ready');
await subscriber.subscribe('finite-events');
server!.block();
await waitUntil(() => !ready());
await sleep(2000);
server!.restore();
await sleep(500);
expect(ready()).toBe(false);
expect(clients!.ioredisClient!.status).not.toBe('ready');
expect(subscriber.status).not.toBe('ready');
expect(
isRedisReady({
useRedis: true,
ioredis: { status: 'ready' },
keyv: clients!.keyvRedisClient,
}),
).toBe(false);
}, 15_000);
});

View file

@ -0,0 +1,220 @@
import net from 'node:net';
import Redis from 'ioredis';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { once } from 'node:events';
import { spawn } from 'node:child_process';
import { mkdtemp, rm } from 'node:fs/promises';
import type { ChildProcess } from 'node:child_process';
import type { Socket } from 'node:net';
export const sleep = (ms: number): Promise<void> =>
new Promise((resolve) => setTimeout(resolve, ms));
export async function waitUntil(predicate: () => boolean, timeout = 5000): Promise<void> {
const deadline = Date.now() + timeout;
while (!predicate()) {
if (Date.now() >= deadline) {
throw new Error('Timed out waiting for Redis state');
}
await sleep(20);
}
}
async function freePort(): Promise<number> {
const server = net.createServer();
try {
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const address = server.address();
if (!address || typeof address === 'string') {
throw new Error('Expected a TCP address');
}
return address.port;
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
}
/** Real Redis behind a TCP fault boundary, including the port advertised to Cluster clients. */
export async function startRedis(cluster: boolean) {
const port = await freePort();
let busPort = await freePort();
while (busPort === port) {
busPort = await freePort();
}
const sockets = new Set<Socket>();
let blocked = false;
let dropReplies = false;
let interruptSubscription = false;
let interruptedSubscriptions = 0;
const proxy = net.createServer((socket) => {
if (blocked) {
socket.destroy();
return;
}
const upstream = net.connect(port, '127.0.0.1');
for (const connection of [socket, upstream]) {
sockets.add(connection);
connection.on('error', () => connection.destroy());
connection.on('close', () => {
sockets.delete(connection);
socket.destroy();
upstream.destroy();
});
}
socket.pipe(upstream);
let replyTail = '';
const subscribeReply = '$9\r\nsubscribe\r\n';
upstream.on('data', (chunk: Buffer) => {
const reply = replyTail + chunk.toString('latin1');
replyTail = reply.slice(-subscribeReply.length);
if (interruptSubscription && reply.includes(subscribeReply)) {
interruptSubscription = false;
interruptedSubscriptions++;
socket.destroy();
upstream.destroy();
return;
}
if (!dropReplies) {
socket.write(chunk);
}
});
});
let dir: string | undefined;
let child: ChildProcess | undefined;
let childClosed: Promise<void> | undefined;
let admin: Redis | undefined;
let startupError: Error | undefined;
const close = async (): Promise<void> => {
admin?.disconnect();
for (const socket of sockets) {
socket.destroy();
}
const proxyClosed = new Promise<void>((resolve) => proxy.close(() => resolve()));
if (child?.pid && child.exitCode === null && child.signalCode === null) {
child.kill('SIGTERM');
}
await Promise.all([proxyClosed, childClosed]);
if (dir) {
await rm(dir, { recursive: true, force: true });
}
};
try {
proxy.listen(0, '127.0.0.1');
await once(proxy, 'listening');
const address = proxy.address();
if (!address || typeof address === 'string') {
throw new Error('Expected a TCP proxy address');
}
dir = await mkdtemp(join(tmpdir(), 'redis-outage-'));
const binary = process.env.REDIS_SERVER_BIN ?? 'redis-server';
child = spawn(
binary,
[
'--bind',
'127.0.0.1',
'--port',
String(port),
'--dir',
dir,
'--save',
'',
'--appendonly',
'no',
'--unixsocket',
'',
...(cluster
? [
'--cluster-enabled',
'yes',
'--cluster-config-file',
'nodes.conf',
'--cluster-port',
String(busPort),
'--cluster-announce-ip',
'127.0.0.1',
'--cluster-announce-port',
String(address.port),
'--cluster-announce-bus-port',
String(busPort),
]
: []),
],
{ stdio: ['ignore', 'pipe', 'pipe'] },
);
let stderr = '';
child.stdout?.resume();
child.stderr?.on('data', (chunk: Buffer) => {
stderr = (stderr + chunk.toString()).slice(-4096);
});
child.on('error', (error: Error) => {
startupError = new Error(`Unable to start Redis (${binary}): ${error.message}`, {
cause: error,
});
});
childClosed = new Promise<void>((resolve) => {
child!.once('close', (code, signal) => {
startupError ??= new Error(
`Redis exited before becoming ready (code=${code}, signal=${signal}): ${stderr.trim()}`,
);
resolve();
});
});
admin = new Redis(port, '127.0.0.1', {
retryStrategy: () => 20,
enableReadyCheck: false,
maxRetriesPerRequest: 3,
keyPrefix: '',
});
admin.on('error', () => undefined);
await waitUntil(() => {
if (startupError) {
throw startupError;
}
return admin?.status === 'ready';
});
if (cluster) {
await admin.cluster('ADDSLOTSRANGE', '0', '16383');
const deadline = Date.now() + 5000;
while (!((await admin.cluster('INFO')) as string).includes('cluster_state:ok')) {
if (startupError) {
throw startupError;
}
if (Date.now() > deadline) {
throw new Error('Redis cluster did not become ready');
}
await sleep(50);
}
}
return {
url: `redis://127.0.0.1:${address.port}`,
admin,
dropReplies: () => {
dropReplies = true;
},
interruptNextSubscription: () => {
interruptSubscription = true;
},
get interruptedSubscriptions() {
return interruptedSubscriptions;
},
block: () => {
blocked = true;
for (const socket of sockets) {
socket.destroy();
}
},
restore: () => {
blocked = false;
dropReplies = false;
},
close,
};
} catch (error) {
const failure = startupError ?? error;
await close();
throw failure;
}
}

View file

@ -0,0 +1,97 @@
import net from 'node:net';
import Redis from 'ioredis';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import fs from 'node:fs/promises';
import childProcess from 'node:child_process';
import { startRedis } from './outage.helper';
describe.each([false, true])('Redis outage fixture startup (cluster=%s)', (cluster) => {
let env: NodeJS.ProcessEnv;
let dir: string;
let servers: jest.SpiedFunction<typeof net.createServer>;
let directories: jest.SpiedFunction<typeof fs.mkdtemp>;
let disconnect: jest.SpiedFunction<typeof Redis.prototype.disconnect>;
let spawned: jest.SpiedFunction<typeof childProcess.spawn>;
beforeAll(async () => {
dir = await fs.mkdtemp(join(tmpdir(), 'redis-fixture-test-'));
await fs.writeFile(
join(dir, 'exits'),
`#!${process.execPath}\nprocess.stderr.write('fixture startup refused\\n', () => process.exit(23));\n`,
{ mode: 0o755 },
);
await fs.writeFile(
join(dir, 'idle'),
`#!${process.execPath}\nsetInterval(() => undefined, 1000);\n`,
{ mode: 0o755 },
);
});
beforeEach(() => {
env = { ...process.env };
servers = jest.spyOn(net, 'createServer');
directories = jest.spyOn(fs, 'mkdtemp');
disconnect = jest.spyOn(Redis.prototype, 'disconnect');
spawned = jest.spyOn(childProcess, 'spawn');
});
afterEach(() => {
process.env = env;
jest.restoreAllMocks();
});
afterAll(async () => {
await fs.rm(dir, { recursive: true, force: true });
});
const expectServersClosed = (): void => {
expect(servers).toHaveBeenCalledTimes(3);
for (const result of servers.mock.results) {
if (result.type === 'return') {
expect(result.value.listening).toBe(false);
}
}
};
const expectCleanup = async (): Promise<void> => {
expectServersClosed();
expect(disconnect).toHaveBeenCalledTimes(1);
expect(directories).toHaveBeenCalledTimes(1);
const directory: string = await directories.mock.results[0].value;
await expect(fs.access(directory)).rejects.toMatchObject({ code: 'ENOENT' });
};
it('rejects a missing binary promptly and closes its proxy, admin, and directory', async () => {
const binary = join(dir, 'missing-redis-server');
process.env.REDIS_SERVER_BIN = binary;
await expect(startRedis(cluster)).rejects.toThrow(
`Unable to start Redis (${binary}): spawn ${binary} ENOENT`,
);
await expectCleanup();
}, 3000);
it('reports an early exit with its code and stderr, then cleans up', async () => {
process.env.REDIS_SERVER_BIN = join(dir, 'exits');
await expect(startRedis(cluster)).rejects.toThrow(
'Redis exited before becoming ready (code=23, signal=null): fixture startup refused',
);
await expectCleanup();
}, 3000);
it('terminates a live child and removes its resources when readiness times out', async () => {
process.env.REDIS_SERVER_BIN = join(dir, 'idle');
await expect(startRedis(cluster)).rejects.toThrow('Timed out waiting for Redis state');
expect(spawned).toHaveBeenCalledTimes(1);
expect(spawned.mock.results[0].value.signalCode).toBe('SIGTERM');
await expectCleanup();
});
it('closes the proxy if allocating the Redis directory fails', async () => {
const error = new Error('fixture directory unavailable');
directories.mockRejectedValueOnce(error);
await expect(startRedis(cluster)).rejects.toBe(error);
expectServersClosed();
expect(disconnect).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,33 @@
jest.mock('@librechat/data-schemas', () => ({ logger: { warn: jest.fn(), error: jest.fn() } }));
jest.mock('~/utils', () => ({
...jest.requireActual('~/utils/math'),
...jest.requireActual('~/utils/common'),
}));
describe('Redis retry configuration', () => {
const original = process.env.REDIS_RETRY_MAX_ATTEMPTS;
afterEach(() => {
if (original === undefined) {
delete process.env.REDIS_RETRY_MAX_ATTEMPTS;
} else {
process.env.REDIS_RETRY_MAX_ATTEMPTS = original;
}
});
it.each([
[undefined, 0],
['0', 0],
['5', 5],
])('reads %s as %s attempts', (value, expected) => {
if (value === undefined) {
delete process.env.REDIS_RETRY_MAX_ATTEMPTS;
} else {
process.env.REDIS_RETRY_MAX_ATTEMPTS = value;
}
jest.isolateModules(() => {
const { cacheConfig } = jest.requireActual<typeof import('./cacheConfig')>('./cacheConfig');
expect(cacheConfig.REDIS_RETRY_MAX_ATTEMPTS).toBe(expected);
});
});
});

View file

@ -174,7 +174,7 @@ const cacheConfig: {
/** Max delay between reconnection attempts in ms */
REDIS_RETRY_MAX_DELAY: math(process.env.REDIS_RETRY_MAX_DELAY, 3000),
/** Max number of reconnection attempts (0 = infinite) */
REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 10),
REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 0),
/** Connection timeout in ms */
REDIS_CONNECT_TIMEOUT: math(process.env.REDIS_CONNECT_TIMEOUT, 10000),
/** Min spacing in ms between reconnects forced by READONLY replies after a failover */

View file

@ -7,6 +7,7 @@ import type { RedisClientType, RedisClusterType } from '@redis/client';
import type { Redis, Cluster } from 'ioredis';
import type { ReadonlyRecoveryHandler } from './recovery';
import { createReadonlyRecovery, isReadonlyReplicaError } from './recovery';
import { createRedisRetryStrategy } from './retry';
import { startRedisHeartbeat } from './heartbeat';
import { cacheConfig } from './cacheConfig';
@ -48,22 +49,13 @@ if (cacheConfig.USE_REDIS) {
tls: useTls ? { ca: ca ?? undefined } : undefined,
keyPrefix: `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR}`,
maxListeners: cacheConfig.REDIS_MAX_LISTENERS,
retryStrategy: (times: number) => {
if (
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
) {
logger.error(
`ioredis giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
);
return null;
}
const base = Math.min(Math.pow(2, times) * 50, cacheConfig.REDIS_RETRY_MAX_DELAY);
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
logger.info(`ioredis reconnecting... attempt ${times}, delay ${delay}ms`);
return delay;
},
retryStrategy: createRedisRetryStrategy({
label: 'ioredis',
maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS,
maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY,
initialDelay: 50,
log: (message) => logger.error(message),
}),
reconnectOnError: (err: Error) => {
if (isReadonlyReplicaError(err)) {
logger.warn('ioredis reconnecting due to READONLY error');
@ -73,7 +65,9 @@ if (cacheConfig.USE_REDIS) {
},
enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
maxRetriesPerRequest: 3,
// A lost reply does not prove a write failed. Reconnect without replaying it.
maxRetriesPerRequest: 0,
autoResendUnfulfilledCommands: false,
keepAlive: cacheConfig.REDIS_KEEP_ALIVE,
};
@ -91,32 +85,25 @@ if (cacheConfig.USE_REDIS) {
}
: {}),
redisOptions,
clusterRetryStrategy: (times: number) => {
if (
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
) {
logger.error(
`ioredis cluster giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
);
return null;
}
const base = Math.min(Math.pow(2, times) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
logger.info(`ioredis cluster reconnecting... attempt ${times}, delay ${delay}ms`);
return delay;
},
// Keep MOVED/ASK routing, but never resend a command after losing its reply.
retryDelayOnFailover: 0,
clusterRetryStrategy: createRedisRetryStrategy({
label: 'ioredis cluster',
maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS,
maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY,
initialDelay: 100,
log: (message) => logger.error(message),
}),
enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
},
);
ioredisClient.on('error', (err) => {
logger.error('ioredis client error:', err);
logger.debug('ioredis client error:', err);
});
ioredisClient.on('connect', () => {
logger.info('ioredis client connected');
logger.debug('ioredis client connected');
});
ioredisClient.on('ready', () => {
@ -124,11 +111,11 @@ if (cacheConfig.USE_REDIS) {
});
ioredisClient.on('reconnecting', (delay: number) => {
logger.info(`ioredis client reconnecting in ${delay}ms`);
logger.debug(`ioredis client reconnecting in ${delay}ms`);
});
ioredisClient.on('close', () => {
logger.warn('ioredis client connection closed');
logger.debug('ioredis client connection closed');
});
/** Deadline-bounded keepalive PINGs; an unanswered probe forces a reconnect. */
@ -201,6 +188,14 @@ if (cacheConfig.USE_REDIS) {
* ** WARNING ** Keyv Redis client does not support Prefix like ioredis above.
* The prefix feature will be handled by the Keyv-Redis store in cacheFactory.js
*/
const retry = createRedisRetryStrategy({
label: '@keyv/redis client',
maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS,
maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY,
initialDelay: 100,
firstAttempt: 0,
log: (message) => logger.error(message),
});
const redisOptions: Record<string, unknown> = {
username,
password,
@ -208,22 +203,8 @@ if (cacheConfig.USE_REDIS) {
...(isRedisCluster ? { tls: useTls } : {}),
...(ca ? { ca } : {}),
connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
reconnectStrategy: (retries: number) => {
if (
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
retries > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
) {
logger.error(
`@keyv/redis client giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
);
return new Error('Max reconnection attempts reached');
}
const base = Math.min(Math.pow(2, retries) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
logger.info(`@keyv/redis reconnecting... attempt ${retries}, delay ${delay}ms`);
return delay;
},
reconnectStrategy: (retries: number) =>
retry(retries) ?? new Error('Max reconnection attempts reached'),
},
disableOfflineQueue: !cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
...(cacheConfig.REDIS_PING_INTERVAL > 0
@ -264,11 +245,11 @@ if (cacheConfig.USE_REDIS) {
}
keyvRedisClient.on('error', (err) => {
logger.error('@keyv/redis client error:', err);
logger.debug('@keyv/redis client error:', err);
});
keyvRedisClient.on('connect', () => {
logger.info('@keyv/redis client connected');
logger.debug('@keyv/redis client connected');
});
keyvRedisClient.on('ready', () => {
@ -276,11 +257,11 @@ if (cacheConfig.USE_REDIS) {
});
keyvRedisClient.on('reconnecting', () => {
logger.info('@keyv/redis client reconnecting...');
logger.debug('@keyv/redis client reconnecting...');
});
keyvRedisClient.on('disconnect', () => {
logger.warn('@keyv/redis client disconnected');
logger.debug('@keyv/redis client disconnected');
});
// Start connection immediately and settle the gate created before client initialization.

View file

@ -1,6 +1,7 @@
import IoRedis from 'ioredis';
import type { RedisClientType } from '@redis/client';
import { duplicateIoRedisClient, scanKeys } from './redisUtils';
import type { Redis, Cluster } from 'ioredis';
import { createIoRedisSubscriber, duplicateIoRedisClient, scanKeys } from './redisUtils';
describe('duplicateIoRedisClient', () => {
it('applies overrides to a single-node duplicate', () => {
@ -81,6 +82,40 @@ describe('duplicateIoRedisClient', () => {
});
});
describe.each([false, true])('createIoRedisSubscriber (cluster=%s)', (cluster) => {
it('retains subscriber commands without changing the publisher retry policy or options', () => {
const redisOptions = {
maxRetriesPerRequest: 0,
autoResendUnfulfilledCommands: false,
keyPrefix: 'subscriber-test::',
};
const publisher = cluster
? new IoRedis.Cluster([{ host: '127.0.0.1', port: 6379 }], {
lazyConnect: true,
redisOptions,
})
: new IoRedis({ ...redisOptions, lazyConnect: true });
const subscriber = createIoRedisSubscriber(publisher, 'test-subscriber');
const options = subscriber.isCluster
? (subscriber as Cluster).options.redisOptions
: (subscriber as Redis).options;
const publisherOptions = publisher.isCluster
? (publisher as Cluster).options.redisOptions
: (publisher as Redis).options;
try {
expect(options).toMatchObject({
maxRetriesPerRequest: null,
autoResendUnfulfilledCommands: true,
keyPrefix: redisOptions.keyPrefix,
});
expect(publisherOptions).toMatchObject(redisOptions);
} finally {
subscriber.disconnect();
publisher.disconnect();
}
});
});
describe('scanKeys', () => {
it('flattens node-redis v5 scan pages', async () => {
const client = {

View file

@ -53,19 +53,36 @@ export function duplicateIoRedisClient(
* The heartbeat is the traffic the shared client gets for free.
*/
export function createIoRedisSubscriber(client: Redis | Cluster, label: string): Redis | Cluster {
const subscriber = duplicateIoRedisClient(client);
// Retain idempotent SUBSCRIBE/PING commands across lost acknowledgements. Publishers
// keep their fail-fast policy so uncertain application writes are never replayed.
const redisOptions: RedisOptions = {
maxRetriesPerRequest: null,
autoResendUnfulfilledCommands: true,
};
const subscriber = duplicateIoRedisClient(
client,
client.isCluster
? { redisOptions: { ...(client as Cluster).options.redisOptions, ...redisOptions } }
: redisOptions,
);
subscriber.on('error', (error: Error) => {
logger.error(`${label} error:`, error);
});
subscriber.on('ready', () => {
logger.info(`${label} ready`);
});
startRedisHeartbeat({
const stopHeartbeat = startRedisHeartbeat({
client: subscriber,
intervalMs: cacheConfig.REDIS_SUBSCRIBER_PING_INTERVAL * 1000,
timeoutMs: cacheConfig.REDIS_PING_TIMEOUT,
label,
});
// An already-closed reconnecting socket need not emit another `end`.
const disconnect = subscriber.disconnect.bind(subscriber);
subscriber.disconnect = (reconnect?: boolean): void => {
if (reconnect !== true) stopHeartbeat();
disconnect(reconnect);
};
return subscriber;
}

66
packages/api/src/cache/retry.spec.ts vendored Normal file
View file

@ -0,0 +1,66 @@
import { createRedisRetryStrategy } from './retry';
describe('Redis reconnect policy', () => {
const log = jest.fn();
const options = {
label: 'Redis',
maxAttempts: 0,
maxDelay: 3000,
initialDelay: 50,
log,
};
beforeEach(() => log.mockClear());
it('keeps retrying for the lifetime of the process without overflowing the backoff', () => {
const retry = createRedisRetryStrategy(options);
for (const attempt of [1, 10, 100, 1024, 100_000, Number.MAX_SAFE_INTEGER]) {
const delay = retry(attempt);
expect(Number.isFinite(delay)).toBe(true);
expect(delay).toBeGreaterThan(0);
expect(delay).toBeLessThanOrEqual(options.maxDelay);
}
});
it('retains jitter once the exponential delay has reached its cap', () => {
const random = jest.spyOn(Math, 'random');
const retry = createRedisRetryStrategy(options);
random.mockReturnValueOnce(0).mockReturnValueOnce(0.999);
expect(retry(10_000)).toBe(3000);
expect(retry(10_000)).toBe(2001);
});
it.each([50, 100])(
'grows the existing %sms backoff and respects a smaller cap',
(initialDelay) => {
jest.spyOn(Math, 'random').mockReturnValue(0);
const retry = createRedisRetryStrategy({ ...options, initialDelay });
expect(retry(1)).toBe(initialDelay * 2);
expect(retry(2)).toBe(initialDelay * 4);
expect(createRedisRetryStrategy({ ...options, initialDelay, maxDelay: 1 })(100_000)).toBe(1);
},
);
it.each([0, 1])('preserves finite overrides for a client counting from %s', (firstAttempt) => {
const retry = createRedisRetryStrategy({ ...options, maxAttempts: 2, firstAttempt });
for (let attempt = firstAttempt; attempt <= 2; attempt++) {
expect(retry(attempt)).not.toBeNull();
}
expect(retry(3)).toBeNull();
expect(log).toHaveBeenLastCalledWith('Redis giving up after 2 reconnection attempts');
});
it.each([0, 1])(
'logs the first and every tenth failure for a client counting from %s',
(firstAttempt) => {
const retry = createRedisRetryStrategy({ ...options, firstAttempt });
for (let attempt = firstAttempt; attempt < firstAttempt + 100; attempt++) {
retry(attempt);
}
expect(log).toHaveBeenCalledTimes(11);
expect(log.mock.calls[0][0]).toContain('attempt 1,');
expect(log.mock.calls[1][0]).toContain('attempt 10,');
expect(log.mock.calls[10][0]).toContain('attempt 100,');
},
);
});

36
packages/api/src/cache/retry.ts vendored Normal file
View file

@ -0,0 +1,36 @@
type RedisRetryOptions = {
label: string;
maxAttempts: number;
maxDelay: number;
initialDelay: number;
firstAttempt?: number;
log: (message: string) => void;
};
/** Reconnect transport only; callers still own failed commands and their side effects. */
export function createRedisRetryStrategy({
label,
maxAttempts,
maxDelay,
initialDelay,
firstAttempt = 1,
log,
}: RedisRetryOptions): (attempt: number) => number | null {
const maxExponent = Math.max(0, Math.ceil(Math.log2(maxDelay / initialDelay)));
return (attempt) => {
if (maxAttempts > 0 && attempt > maxAttempts) {
log(`${label} giving up after ${maxAttempts} reconnection attempts`);
return null;
}
// Saturate the exponent before computing it, even after months of reconnects.
const exponent = Math.min(attempt, maxExponent);
const base = Math.min(initialDelay * 2 ** exponent, maxDelay);
// Keep jitter at the cap rather than clipping every retry to the same delay.
const delay = base - Math.floor(Math.random() * Math.min(base / 2, 1000));
const failure = attempt - firstAttempt + 1;
if (failure === 1 || failure % 10 === 0) {
log(`${label} reconnecting... attempt ${failure}, delay ${delay}ms`);
}
return delay;
};
}

View file

@ -42,6 +42,210 @@ function deliverSequencedMessage(
}
describe('RedisEventTransport', () => {
it.each([
{ mode: 'ordinary', removal: 'unsubscribe' },
{ mode: 'frontier', removal: 'unsubscribe' },
{ mode: 'replay', removal: 'unsubscribe' },
{ mode: 'ordinary', removal: 'cleanup' },
{ mode: 'frontier', removal: 'cleanup' },
{ mode: 'replay', removal: 'cleanup' },
] as const)(
'rejects $mode readiness first awaited after $removal and destroy',
async ({ mode, removal }) => {
const publisher = createMockPublisher();
const subscriber = createMockSubscriber();
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
const transport = new RedisEventTransport(
publisher as unknown as Redis,
subscriber as unknown as Redis,
);
const options =
mode === 'replay'
? { replay: { items: 10, bytes: 4096, ttlMs: 1000 } }
: { deferSequenceDelivery: true, captureSequenceFrontier: mode === 'frontier' };
const attachment = transport.subscribe(
'closed-before-await',
{ onChunk: jest.fn() },
options,
);
if (removal === 'unsubscribe') attachment.unsubscribe();
else transport.cleanup('closed-before-await');
transport.destroy();
await expect(attachment.ready).rejects.toThrow('Redis event transport is closed.');
expect(publisher.eval).not.toHaveBeenCalled();
expect(transport.getTrackedStreamIds()).toEqual([]);
},
);
it.each(['abort', 'preempt'] as const)(
'rejects a pending %s registration explicitly awaited after destroy',
async (kind) => {
const subscriber = createMockSubscriber();
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
const transport = new RedisEventTransport(
createMockPublisher() as unknown as Redis,
subscriber as unknown as Redis,
);
const registration =
kind === 'abort'
? transport.onAbort('closed-control', jest.fn())
: transport.onPreempt('closed-control', jest.fn());
transport.cleanup('closed-control');
transport.destroy();
await expect(registration).rejects.toThrow('Redis event transport is closed.');
expect(transport.getTrackedStreamIds()).toEqual([]);
},
);
it('rejects both frontier readiness and synchronization when removed before their first await', async () => {
const publisher = createMockPublisher();
const subscriber = createMockSubscriber();
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
const transport = new RedisEventTransport(
publisher as unknown as Redis,
subscriber as unknown as Redis,
);
const attachment = transport.subscribe(
'closed-frontier-sync',
{ onChunk: jest.fn() },
{
deferSequenceDelivery: true,
captureSequenceFrontier: true,
},
);
const synchronized = attachment.syncReorderBuffer?.();
attachment.unsubscribe();
transport.destroy();
await Promise.all([
expect(attachment.ready).rejects.toThrow('Redis event transport is closed.'),
expect(synchronized).rejects.toThrow('Redis event transport is closed.'),
]);
expect(publisher.eval).not.toHaveBeenCalled();
});
it.each(['frontier', 'replay'] as const)(
'preserves the original %s datastore readiness error',
async (mode) => {
const publisher = createMockPublisher();
const failure = new Error('fixture EVAL failure');
publisher.eval.mockRejectedValueOnce(failure);
const transport = new RedisEventTransport(
publisher as unknown as Redis,
createMockSubscriber() as unknown as Redis,
);
const options =
mode === 'replay'
? { replay: { items: 10, bytes: 4096, ttlMs: 1000 } }
: { deferSequenceDelivery: true, captureSequenceFrontier: true };
const attachment = transport.subscribe('failed-readiness', { onChunk: jest.fn() }, options);
try {
await expect(attachment.ready).rejects.toBe(failure);
} finally {
attachment.unsubscribe();
transport.destroy();
}
},
);
it('preserves a GET failure even when cleanup removes the stream before synchronization settles', async () => {
const publisher = createMockPublisher();
let failRead!: (error: Error) => void;
publisher.get.mockImplementationOnce(
() =>
new Promise((_resolve, reject) => {
failRead = reject;
}),
);
const transport = new RedisEventTransport(
publisher as unknown as Redis,
createMockSubscriber() as unknown as Redis,
);
const attachment = transport.subscribe('removed-sync', { onChunk: jest.fn() });
await attachment.ready;
const synchronized = attachment.syncReorderBuffer?.();
transport.cleanup('removed-sync');
const failure = new Error('fixture GET failure');
failRead(failure);
await expect(synchronized).rejects.toBe(failure);
transport.destroy();
});
it('cancels a pending attachment locally and disconnects its subscriber only once', async () => {
jest.useFakeTimers();
const publisher = createMockPublisher();
const subscriber = { ...createMockSubscriber(), disconnect: jest.fn() };
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
const transport = new RedisEventTransport(
publisher as unknown as Redis,
subscriber as unknown as Redis,
);
try {
const attachment = transport.subscribe('unacknowledged-close', { onChunk: jest.fn() });
const ready = attachment.ready?.then(
() => 'ready',
(error: unknown) => error,
);
transport.destroy();
transport.destroy();
expect(await ready).toBeInstanceOf(Error);
expect(subscriber.disconnect).toHaveBeenCalledTimes(1);
expect(subscriber.unsubscribe).not.toHaveBeenCalled();
jest.runAllTicks();
expect(jest.getTimerCount()).toBe(0);
expect(() => transport.subscribe('after-close', { onChunk: jest.fn() })).toThrow(
'Redis event transport is closed.',
);
} finally {
transport.destroy();
jest.useRealTimers();
}
});
it('does not publish a frontier or reopen a channel after destruction during subscribe', async () => {
const publisher = createMockPublisher();
const subscriber = createMockSubscriber();
let acknowledge!: () => void;
subscriber.subscribe.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
acknowledge = resolve;
}),
);
const transport = new RedisEventTransport(
publisher as unknown as Redis,
subscriber as unknown as Redis,
);
const attachment = transport.subscribe(
'destroyed-attachment',
{ onChunk: jest.fn() },
{
deferSequenceDelivery: true,
captureSequenceFrontier: true,
},
);
const ready = attachment.ready?.then(
() => 'ready',
(error: unknown) => error,
);
transport.destroy();
acknowledge();
expect(await ready).toBeInstanceOf(Error);
expect(publisher.eval).not.toHaveBeenCalled();
expect(transport.getTrackedStreamIds()).toEqual([]);
});
it('delivers tagged events and preserves legacy in-memory callback arity', () => {
const transport = new InMemoryEventTransport();
const onChunk = jest.fn();
@ -2234,7 +2438,7 @@ describe('RedisEventTransport', () => {
);
const streamId = 'reorder-abort-listener-reuse-test';
transport.onAbort(streamId, () => {});
const disposeAbort = await transport.onAbort(streamId, () => {});
const messageHandler = getMessageHandler(mockSubscriber);
const channel = `stream:{${streamId}}:events`;
@ -2255,6 +2459,7 @@ describe('RedisEventTransport', () => {
expect(secondRunChunks.map((chunk) => (chunk as { index: number }).index)).toEqual([0]);
disposeAbort();
transport.destroy();
});
});

View file

@ -1127,7 +1127,7 @@ describe('RedisEventTransport Integration Tests', () => {
let abortReceived = false;
// Register abort callback
transport.onAbort(streamId, () => {
const disposeAbort = await transport.onAbort(streamId, () => {
abortReceived = true;
});
@ -1142,6 +1142,7 @@ describe('RedisEventTransport Integration Tests', () => {
expect(abortReceived).toBe(true);
disposeAbort();
transport.destroy();
subscriber.disconnect();
});
@ -1307,10 +1308,10 @@ describe('RedisEventTransport Integration Tests', () => {
let callback2Called = false;
// Multiple abort callbacks
transport.onAbort(streamId, () => {
const disposeFirst = await transport.onAbort(streamId, () => {
callback1Called = true;
});
transport.onAbort(streamId, () => {
const disposeSecond = await transport.onAbort(streamId, () => {
callback2Called = true;
});
@ -1323,6 +1324,8 @@ describe('RedisEventTransport Integration Tests', () => {
expect(callback1Called).toBe(true);
expect(callback2Called).toBe(true);
disposeFirst();
disposeSecond();
transport.destroy();
subscriber.disconnect();
});
@ -1341,7 +1344,7 @@ describe('RedisEventTransport Integration Tests', () => {
const streamId = `abort-cleanup-${Date.now()}`;
let abortReceived = false;
transport.onAbort(streamId, () => {
const disposeAbort = await transport.onAbort(streamId, () => {
abortReceived = true;
});
@ -1349,6 +1352,7 @@ describe('RedisEventTransport Integration Tests', () => {
// Cleanup the stream
transport.cleanup(streamId);
disposeAbort();
// Emit abort after cleanup
transport.emitAbort(streamId);

View file

@ -77,7 +77,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => {
transport.destroy();
});
test('abort callback survives across reconnect cycles', () => {
test('abort callback survives across reconnect cycles', async () => {
const mockPublisher = createMockPublisher();
const mockSubscriber = {
on: jest.fn(),
@ -94,7 +94,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => {
let abortCallbackFired = false;
// Register abort callback (simulates what createJob does)
transport.onAbort(streamId, () => {
const disposeAbort = await transport.onAbort(streamId, () => {
abortCallbackFired = true;
});
@ -115,6 +115,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => {
// Abort callback should fire — it was preserved across the reconnect
expect(abortCallbackFired).toBe(true);
disposeAbort();
sub2.unsubscribe();
transport.destroy();
});

View file

@ -251,6 +251,7 @@ interface SubscriptionFrontierWaiter {
interface ChannelSubscriptionState {
ready: Promise<void>;
cancel?: () => void;
phase: 'pending' | 'active';
/** A timed-out predecessor became active while this replacement was pending. */
fallbackActive: boolean;
@ -304,6 +305,12 @@ interface StreamSubscribers {
* ```
*/
export class RedisEventTransport implements IEventTransport {
private destroyed = false;
private assertOpen(): void {
if (this.destroyed) throw new Error('Redis event transport is closed.');
}
/** Redis client for publishing events */
private publisher: Redis | Cluster;
/** Redis client for subscribing to events (separate connection required) */
@ -312,6 +319,8 @@ export class RedisEventTransport implements IEventTransport {
private streams = new Map<string, StreamSubscribers>();
/** Track channel subscription state: resolved promise = active, pending = in-flight */
private channelSubscriptions = new Map<string, ChannelSubscriptionState>();
/** Pending waits outlive channel removal until they settle or the transport closes. */
private pendingSubscriptions = new Set<ChannelSubscriptionState>();
/** Counter for generating unique subscriber IDs */
private subscriberIdCounter = 0;
/** Coalescable chunk publications awaiting their window flush, per stream */
@ -338,6 +347,7 @@ export class RedisEventTransport implements IEventTransport {
}
private getOrCreateStreamState(streamId: string): StreamSubscribers {
this.assertOpen();
const existing = this.streams.get(streamId);
if (existing) {
return existing;
@ -353,6 +363,7 @@ export class RedisEventTransport implements IEventTransport {
streamId: string,
replay = false,
): Promise<number | ReplaySnapshot> {
this.assertOpen();
const subscriptionFrontierId = randomUUID();
let operationTimeout: ReturnType<typeof setTimeout> | undefined;
const observed = new Promise<void>((resolve, reject) => {
@ -396,6 +407,7 @@ export class RedisEventTransport implements IEventTransport {
operationTimeout.unref?.();
});
const [raw] = await Promise.race([operation, operationDeadline]);
this.assertOpen();
const counter = replay ? (raw as [string, string[]])[0] : raw;
const parsed = counter != null ? parseInt(String(counter), 10) : 0;
const frontier = Number.isNaN(parsed) ? 0 : parsed;
@ -441,7 +453,7 @@ export class RedisEventTransport implements IEventTransport {
// Set up message handler for all subscriptions
this.subscriber.on('message', (channel: string, message: string) => {
this.handleMessage(channel, message);
if (!this.destroyed) this.handleMessage(channel, message);
});
}
@ -677,6 +689,7 @@ export class RedisEventTransport implements IEventTransport {
}
private ensureChannelSubscription(streamId: string): Promise<void> {
this.assertOpen();
const channel = CHANNELS.events(streamId);
const existing = this.channelSubscriptions.get(channel);
if (existing) {
@ -692,6 +705,10 @@ export class RedisEventTransport implements IEventTransport {
fallbackActive: false,
};
const deadline = new Promise<never>((_, reject) => {
state.cancel = () => {
clearTimeout(timeout);
reject(new Error('Redis event transport is closed.'));
};
timeout = setTimeout(() => {
expired = true;
reject(new Error(`Timed out synchronizing Redis subscription for ${channel}`));
@ -699,6 +716,7 @@ export class RedisEventTransport implements IEventTransport {
timeout.unref?.();
});
const ready = Promise.race([operation, deadline]).then(() => {
this.assertOpen();
state.phase = 'active';
state.fallbackActive = false;
logger.debug(`[RedisEventTransport] Subscription active for channel ${channel}`);
@ -707,7 +725,7 @@ export class RedisEventTransport implements IEventTransport {
operation.then(
() => {
if (timeout != null) clearTimeout(timeout);
if (expired) {
if (expired && !this.destroyed) {
const current = this.channelSubscriptions.get(channel);
if (current == null || current === state) {
if (current === state) this.channelSubscriptions.delete(channel);
@ -724,13 +742,19 @@ export class RedisEventTransport implements IEventTransport {
},
);
this.channelSubscriptions.set(channel, state);
void ready.catch((err) => {
if (this.channelSubscriptions.get(channel) === state) {
this.channelSubscriptions.delete(channel);
if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel);
}
logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err);
});
this.pendingSubscriptions.add(state);
void ready.then(
() => this.pendingSubscriptions.delete(state),
(err) => {
this.pendingSubscriptions.delete(state);
if (this.destroyed) return;
if (this.channelSubscriptions.get(channel) === state) {
this.channelSubscriptions.delete(channel);
if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel);
}
logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err);
},
);
return ready;
}
@ -1943,11 +1967,15 @@ export class RedisEventTransport implements IEventTransport {
* Destroy all resources.
*/
destroy(): void {
for (const streamId of new Set(
[...this.subscriptionFrontierWaiters.values()].map((waiter) => waiter.streamId),
)) {
this.releaseSubscriptionFrontiers(streamId);
if (this.destroyed) return;
this.destroyed = true;
for (const waiter of this.subscriptionFrontierWaiters.values()) {
clearTimeout(waiter.timeout);
waiter.reject(new Error('Redis event transport is closed.'));
}
this.subscriptionFrontierWaiters.clear();
for (const state of this.pendingSubscriptions) state.cancel?.();
this.pendingSubscriptions.clear();
for (const streamId of this.pendingBatches.keys()) {
this.discardCoalescedChunks(streamId);
}
@ -1969,10 +1997,6 @@ export class RedisEventTransport implements IEventTransport {
state.abortAckWaiters.clear();
}
for (const channel of this.channelSubscriptions.keys()) {
this.subscriber.unsubscribe(channel).catch(() => {});
}
this.channelSubscriptions.clear();
this.streams.clear();