mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 07:23:11 +00:00
🐦🔥 fix: Recover Redis Clients After Extended Outages (#18164)
* 🐦🔥 fix: Recover Redis Clients After Extended Outages * fix: Retain Subscriber Registrations Across Repeated Disconnects * test: Diagnose Redis Fixture Startup Failures * fix: Close Redis Invalidation Connections During Outages * fix: Bound Redis Subscriber Lifetimes During Shutdown * fix: Retain Pending Redis Subscription Cancellation After Removal --------- Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
687af9ac5a
commit
e1fc59c9cf
20 changed files with 1586 additions and 113 deletions
218
packages/api/src/agents/subagentTaskRouting.redis.spec.ts
Normal file
218
packages/api/src/agents/subagentTaskRouting.redis.spec.ts
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import Redis from 'ioredis';
|
||||
import http from 'node:http';
|
||||
import type { SubagentTaskControlHandler } from './subagentTaskRouting';
|
||||
import type { RespServer } from '~/cache/__tests__/resp.helper';
|
||||
import {
|
||||
RedisSubagentTaskControlTransport,
|
||||
SubagentTaskOwnerUnavailableError,
|
||||
} from './subagentTaskRouting';
|
||||
import {
|
||||
setupGracefulShutdown,
|
||||
registerShutdownTask,
|
||||
__resetShutdownStateForTests,
|
||||
} from '~/app/shutdown';
|
||||
import { createIoRedisSubscriber, duplicateIoRedisClient } from '~/cache/redisUtils';
|
||||
import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
jest.mock('~/utils', () => ({
|
||||
...jest.requireActual('~/utils/math'),
|
||||
...jest.requireActual('~/utils/common'),
|
||||
}));
|
||||
|
||||
const handler: SubagentTaskControlHandler = {
|
||||
claim: () => ({ status: 'not_found' }),
|
||||
control: () => ({ status: 'not_found' }),
|
||||
list: () => [],
|
||||
retainsTaskOwnership: () => false,
|
||||
cancelScope: () => 0,
|
||||
};
|
||||
const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGHUP'];
|
||||
|
||||
describe('Redis task-routing lifecycle', () => {
|
||||
let server: RespServer;
|
||||
let shared: Redis;
|
||||
let publisher: Redis;
|
||||
let subscriber: Redis;
|
||||
let transport: RedisSubagentTaskControlTransport;
|
||||
let shutdownExit: jest.SpyInstance | undefined;
|
||||
let originalListeners: Map<NodeJS.Signals, NodeJS.SignalsListener[]>;
|
||||
|
||||
beforeEach(async () => {
|
||||
originalListeners = new Map(signals.map((signal) => [signal, process.listeners(signal)]));
|
||||
__resetShutdownStateForTests();
|
||||
shutdownExit = undefined;
|
||||
server = await startRespServer();
|
||||
shared = new Redis(server.url, {
|
||||
maxRetriesPerRequest: 0,
|
||||
autoResendUnfulfilledCommands: false,
|
||||
retryStrategy: () => 20,
|
||||
});
|
||||
shared.on('error', () => undefined);
|
||||
await waitFor(() => shared.status === 'ready');
|
||||
publisher = duplicateIoRedisClient(shared, { enableOfflineQueue: false }) as Redis;
|
||||
publisher.on('error', () => undefined);
|
||||
subscriber = createIoRedisSubscriber(shared, 'task-lifecycle-subscriber') as Redis;
|
||||
transport = new RedisSubagentTaskControlTransport(publisher, subscriber);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await transport.destroy();
|
||||
subscriber.disconnect();
|
||||
publisher.disconnect();
|
||||
shared.disconnect();
|
||||
await server.close();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
if (shutdownExit) await waitFor(() => shutdownExit!.mock.calls.length === 1, 500);
|
||||
} finally {
|
||||
__resetShutdownStateForTests();
|
||||
for (const signal of signals) {
|
||||
for (const listener of process.listeners(signal)) {
|
||||
if (!originalListeners.get(signal)?.includes(listener)) {
|
||||
process.removeListener(signal, listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
shutdownExit?.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('cancels a readiness wait when destroyed without closing its borrowed publisher', async () => {
|
||||
await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready');
|
||||
await server.close();
|
||||
await waitFor(
|
||||
() => publisher.status === 'reconnecting' && subscriber.status === 'reconnecting',
|
||||
);
|
||||
const publisherDisconnect = jest.spyOn(publisher, 'disconnect');
|
||||
const publisherOnce = jest.spyOn(publisher, 'once');
|
||||
const subscriberOnce = jest.spyOn(subscriber, 'once');
|
||||
let result: unknown;
|
||||
void transport.bind(handler).then(
|
||||
() => {
|
||||
result = 'bound';
|
||||
},
|
||||
(error: unknown) => {
|
||||
result = error;
|
||||
},
|
||||
);
|
||||
|
||||
const readinessListeners = [publisherOnce, subscriberOnce].map((spy) =>
|
||||
spy.mock.calls.filter(([event]) => event === 'ready' || event === 'end'),
|
||||
);
|
||||
let destroyed = false;
|
||||
void transport.destroy().then(() => {
|
||||
destroyed = true;
|
||||
});
|
||||
await waitFor(() => result != null && destroyed, 500);
|
||||
|
||||
expect(result).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
|
||||
for (const [index, client] of [publisher, subscriber].entries()) {
|
||||
for (const [event, listener] of readinessListeners[index]) {
|
||||
expect(client.listeners(event)).not.toContain(listener);
|
||||
}
|
||||
}
|
||||
expect(publisherDisconnect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not subscribe when readiness resolves just before destruction', async () => {
|
||||
await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready');
|
||||
const subscribe = jest.spyOn(subscriber, 'subscribe');
|
||||
const binding = transport.bind(handler).then(
|
||||
() => 'bound',
|
||||
(error: unknown) => error,
|
||||
);
|
||||
|
||||
await transport.destroy();
|
||||
|
||||
expect(await binding).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
|
||||
expect(subscribe).not.toHaveBeenCalled();
|
||||
expect(subscriber.listenerCount('message')).toBe(0);
|
||||
await expect(publisher.ping()).resolves.toBe('PONG');
|
||||
});
|
||||
|
||||
it('cancels an unacknowledged initial subscription on destruction', async () => {
|
||||
await waitFor(() => publisher.status === 'ready' && subscriber.status === 'ready');
|
||||
server.silent = true;
|
||||
let result: unknown;
|
||||
void transport.bind(handler).then(
|
||||
() => {
|
||||
result = 'bound';
|
||||
},
|
||||
(error: unknown) => {
|
||||
result = error;
|
||||
},
|
||||
);
|
||||
await waitFor(() => server.commands.some((frame) => frame[0] === 'subscribe'));
|
||||
|
||||
let destroyed = false;
|
||||
void transport.destroy().then(() => {
|
||||
destroyed = true;
|
||||
});
|
||||
await waitFor(() => result != null && destroyed, 500);
|
||||
|
||||
expect(result).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
|
||||
expect(subscriber.listenerCount('message')).toBe(0);
|
||||
});
|
||||
|
||||
it('stops its heartbeat on intentional disconnect even when the closed socket emits no end', async () => {
|
||||
await transport.bind(handler);
|
||||
await server.close();
|
||||
await waitFor(() => subscriber.status === 'reconnecting');
|
||||
const listeners = subscriber.listenerCount('end');
|
||||
expect(listeners).toBeGreaterThan(0);
|
||||
|
||||
subscriber.disconnect(true);
|
||||
expect(subscriber.listenerCount('end')).toBe(listeners);
|
||||
subscriber.disconnect();
|
||||
|
||||
expect(subscriber.listenerCount('end')).toBe(listeners - 1);
|
||||
});
|
||||
|
||||
it('allows later shutdown tasks to run while Redis is unreachable', async () => {
|
||||
await transport.bind(handler);
|
||||
await server.close();
|
||||
await waitFor(() => subscriber.status === 'reconnecting');
|
||||
const steps: string[] = [];
|
||||
const exit = jest.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
|
||||
shutdownExit = exit;
|
||||
registerShutdownTask(
|
||||
'generation job manager',
|
||||
() => {
|
||||
steps.push('generation');
|
||||
},
|
||||
{ priority: 100 },
|
||||
);
|
||||
registerShutdownTask(
|
||||
'subagent task store',
|
||||
async () => {
|
||||
await transport.destroy();
|
||||
publisher.disconnect();
|
||||
steps.push('task-routing');
|
||||
},
|
||||
{ priority: 90 },
|
||||
);
|
||||
registerShutdownTask(
|
||||
'favorites persistence',
|
||||
() => {
|
||||
steps.push('favorites');
|
||||
},
|
||||
{ priority: -100 },
|
||||
);
|
||||
setupGracefulShutdown(http.createServer());
|
||||
|
||||
const listener = process
|
||||
.listeners('SIGTERM')
|
||||
.find(
|
||||
(entry) => !originalListeners.get('SIGTERM')?.includes(entry),
|
||||
) as NodeJS.SignalsListener;
|
||||
listener('SIGTERM');
|
||||
await waitFor(() => exit.mock.calls.length === 1, 500);
|
||||
|
||||
expect(steps).toEqual(['generation', 'task-routing', 'favorites']);
|
||||
expect(exit).toHaveBeenCalledWith(0);
|
||||
await transport.destroy();
|
||||
});
|
||||
});
|
||||
|
|
@ -184,6 +184,144 @@ function taskHandler(
|
|||
}
|
||||
|
||||
describe('RedisSubagentTaskControlTransport', () => {
|
||||
it('bounds a missing initial SUBSCRIBE acknowledgement and closes only its subscriber', async () => {
|
||||
jest.useFakeTimers();
|
||||
const bus = new FakeRedisBus();
|
||||
const publisher = bus.createClient();
|
||||
const subscriber = bus.createClient();
|
||||
jest.spyOn(subscriber, 'subscribe').mockImplementation(() => new Promise(() => undefined));
|
||||
const transport = new RedisSubagentTaskControlTransport(
|
||||
asRedis(publisher),
|
||||
asRedis(subscriber),
|
||||
);
|
||||
try {
|
||||
const binding = transport.bind(taskHandler()).then(
|
||||
() => 'bound',
|
||||
(error: unknown) => error,
|
||||
);
|
||||
await jest.advanceTimersByTimeAsync(10_000);
|
||||
expect(await binding).toBeInstanceOf(SubagentTaskOwnerUnavailableError);
|
||||
expect(subscriber.disconnected).toBe(true);
|
||||
expect(subscriber.listeners.size).toBe(0);
|
||||
expect(publisher.disconnected).toBe(false);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
} finally {
|
||||
await transport.destroy();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('ignores a message callback queued before destruction', async () => {
|
||||
const bus = new FakeRedisBus();
|
||||
const subscriber = bus.createClient();
|
||||
const transport = new RedisSubagentTaskControlTransport(
|
||||
asRedis(bus.createClient()),
|
||||
asRedis(subscriber),
|
||||
);
|
||||
const control = jest.fn(() => ({ status: 'not_found' as const }));
|
||||
await transport.bind(taskHandler({ control }));
|
||||
const listener = [...subscriber.listeners][0];
|
||||
const channel = [...subscriber.channels][0];
|
||||
await transport.destroy();
|
||||
|
||||
listener(
|
||||
channel,
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
kind: 'request',
|
||||
operation: 'control',
|
||||
requestId: 'late-request',
|
||||
requesterId: 'requester',
|
||||
scopeId: 'scope-1',
|
||||
taskId: 'task-1',
|
||||
invocationId: 'invocation-1',
|
||||
expiresAt: Date.now() + 1000,
|
||||
command: { action: 'queue', message: 'late control' },
|
||||
}),
|
||||
);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(control).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not publish a control reply that settles after destruction', async () => {
|
||||
const bus = new FakeRedisBus();
|
||||
const publisher = bus.createClient();
|
||||
const subscriber = bus.createClient();
|
||||
const transport = new RedisSubagentTaskControlTransport(
|
||||
asRedis(publisher),
|
||||
asRedis(subscriber),
|
||||
);
|
||||
let finish!: (value: SubagentTaskControlResult) => void;
|
||||
const control = jest.fn(
|
||||
() =>
|
||||
new Promise<SubagentTaskControlResult>((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
await transport.bind(taskHandler({ control }));
|
||||
const channel = [...subscriber.channels][0];
|
||||
bus.publish(
|
||||
channel,
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
kind: 'request',
|
||||
operation: 'control',
|
||||
requestId: 'pending-request',
|
||||
requesterId: 'requester',
|
||||
scopeId: 'scope-1',
|
||||
taskId: 'task-1',
|
||||
invocationId: 'invocation-1',
|
||||
expiresAt: Date.now() + 1000,
|
||||
command: { action: 'queue', message: 'pending control' },
|
||||
}),
|
||||
);
|
||||
await Promise.resolve();
|
||||
expect(control).toHaveBeenCalledTimes(1);
|
||||
await transport.destroy();
|
||||
const publish = jest.spyOn(publisher, 'publish');
|
||||
|
||||
finish({ status: 'not_found' });
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(publish).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not issue queued owner refreshes after destruction', async () => {
|
||||
jest.useFakeTimers();
|
||||
const bus = new FakeRedisBus();
|
||||
const publisher = bus.createClient();
|
||||
const transport = new RedisSubagentTaskControlTransport(
|
||||
asRedis(publisher),
|
||||
asRedis(bus.createClient()),
|
||||
{ registrationHeartbeatMs: 1 },
|
||||
);
|
||||
const tasks = Array.from({ length: 36 }, (_, index) => snapshot({ taskId: `task-${index}` }));
|
||||
const releases: Array<() => void> = [];
|
||||
try {
|
||||
await transport.bind(taskHandler({ list: () => tasks }));
|
||||
for (const task of tasks) await transport.registerTask('scope-1', task.taskId, 60_000);
|
||||
bus.registrationHook = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releases.push(resolve);
|
||||
});
|
||||
const evalCommand = jest.spyOn(publisher, 'eval');
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(releases).toHaveLength(32);
|
||||
|
||||
await transport.destroy();
|
||||
for (const release of releases) release();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(evalCommand).toHaveBeenCalledTimes(32);
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
} finally {
|
||||
await transport.destroy();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for the fail-fast publisher before reporting itself bound', async () => {
|
||||
const bus = new FakeRedisBus();
|
||||
const publisher = new EventEmitter() as EventEmitter & { status: string };
|
||||
|
|
|
|||
|
|
@ -355,7 +355,13 @@ function boundedControlResult(result: SubagentTaskControlResult): SubagentTaskCo
|
|||
};
|
||||
}
|
||||
|
||||
async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
|
||||
async function waitForRedisConnectionReady(
|
||||
client: RedisClient,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (signal.aborted) {
|
||||
throw new SubagentTaskOwnerUnavailableError();
|
||||
}
|
||||
if (client.status == null || client.status === 'ready') {
|
||||
return;
|
||||
}
|
||||
|
|
@ -367,6 +373,7 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
|
|||
clearTimeout(timeout);
|
||||
client.off('ready', onReady);
|
||||
client.off('end', onEnd);
|
||||
signal.removeEventListener('abort', onEnd);
|
||||
};
|
||||
const onReady = () => {
|
||||
cleanup();
|
||||
|
|
@ -381,11 +388,14 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
|
|||
reject(new SubagentTaskOwnerUnavailableError());
|
||||
}, DEFAULT_READY_TIMEOUT_MS);
|
||||
timeout.unref?.();
|
||||
signal.addEventListener('abort', onEnd, { once: true });
|
||||
client.once('ready', onReady);
|
||||
client.once('end', onEnd);
|
||||
/** Close the status-check/listener-registration race: ioredis may become ready
|
||||
* synchronously between the check above and installing these listeners. */
|
||||
if (client.status === 'ready') {
|
||||
if (signal.aborted) {
|
||||
onEnd();
|
||||
} else if (client.status === 'ready') {
|
||||
onReady();
|
||||
} else if (client.status === 'end') {
|
||||
onEnd();
|
||||
|
|
@ -397,9 +407,10 @@ async function waitForRedisConnectionReady(client: RedisClient): Promise<void> {
|
|||
|
||||
async function waitForRedisReady(
|
||||
client: RedisClient,
|
||||
signal: AbortSignal,
|
||||
options: { eagerClusterMasters?: boolean } = {},
|
||||
): Promise<void> {
|
||||
await waitForRedisConnectionReady(client);
|
||||
await waitForRedisConnectionReady(client, signal);
|
||||
if (options.eagerClusterMasters !== true || !client.isCluster) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -407,10 +418,41 @@ async function waitForRedisReady(
|
|||
* fail-fast publisher cannot admit requests until every possible write target is
|
||||
* connected; otherwise the first command to a cold shard would be rejected. */
|
||||
await Promise.all(
|
||||
(client as Cluster).nodes('master').map((node) => waitForRedisConnectionReady(node)),
|
||||
(client as Cluster).nodes('master').map((node) => waitForRedisConnectionReady(node, signal)),
|
||||
);
|
||||
}
|
||||
|
||||
/** Bounds the initial SUBSCRIBE acknowledgement and cancels it with its owner. */
|
||||
async function waitForRedisSubscription(
|
||||
operation: Promise<unknown>,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = (): void => {
|
||||
clearTimeout(timeout);
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
};
|
||||
const onAbort = (): void => {
|
||||
cleanup();
|
||||
reject(new SubagentTaskOwnerUnavailableError());
|
||||
};
|
||||
const timeout = setTimeout(onAbort, DEFAULT_READY_TIMEOUT_MS);
|
||||
timeout.unref?.();
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
operation.then(
|
||||
() => {
|
||||
cleanup();
|
||||
resolve();
|
||||
},
|
||||
(error: unknown) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
if (signal.aborted) onAbort();
|
||||
});
|
||||
}
|
||||
|
||||
function isSnapshot(value: unknown): value is SubagentTaskSnapshot {
|
||||
if (value == null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return false;
|
||||
|
|
@ -734,7 +776,9 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
private registrationHeartbeat?: ReturnType<typeof setInterval>;
|
||||
private registrationRefresh?: Promise<void>;
|
||||
private destroyed = false;
|
||||
private readonly lifecycle = new AbortController();
|
||||
|
||||
/** Borrows the publisher; owns the dedicated subscriber and its pending waits. */
|
||||
constructor(
|
||||
private readonly publisher: RedisClient,
|
||||
private readonly subscriber: RedisClient,
|
||||
|
|
@ -764,13 +808,22 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
/** The publisher fails fast instead of queueing commands, so opening HTTP
|
||||
* admission before it is ready would turn healthy startup lag into false
|
||||
* `unavailable` results. Both dedicated connections are part of readiness. */
|
||||
await Promise.all([
|
||||
waitForRedisReady(this.publisher, { eagerClusterMasters: true }),
|
||||
waitForRedisReady(this.subscriber),
|
||||
]);
|
||||
this.subscriber.on('message', this.onMessage);
|
||||
this.ready = this.subscriber.subscribe(this.channel(this.instanceId)).then(() => undefined);
|
||||
await this.ready;
|
||||
try {
|
||||
await Promise.all([
|
||||
waitForRedisReady(this.publisher, this.lifecycle.signal, { eagerClusterMasters: true }),
|
||||
waitForRedisReady(this.subscriber, this.lifecycle.signal),
|
||||
]);
|
||||
if (this.destroyed) throw new SubagentTaskOwnerUnavailableError();
|
||||
this.subscriber.on('message', this.onMessage);
|
||||
this.ready = waitForRedisSubscription(
|
||||
this.subscriber.subscribe(this.channel(this.instanceId)),
|
||||
this.lifecycle.signal,
|
||||
);
|
||||
await this.requireReady();
|
||||
} catch (error) {
|
||||
await this.destroy();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async registerTask(scopeId: string, taskId: string, ttlMs: number): Promise<void> {
|
||||
|
|
@ -963,6 +1016,8 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
return;
|
||||
}
|
||||
this.destroyed = true;
|
||||
this.lifecycle.abort();
|
||||
this.handler = undefined;
|
||||
for (const pending of this.pending.values()) {
|
||||
clearTimeout(pending.retry);
|
||||
clearTimeout(pending.timeout);
|
||||
|
|
@ -979,12 +1034,15 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
this.registrationHeartbeat = undefined;
|
||||
}
|
||||
this.subscriber.off('message', this.onMessage);
|
||||
await this.subscriber.unsubscribe(this.channel(this.instanceId)).catch(() => undefined);
|
||||
this.subscriber.disconnect();
|
||||
}
|
||||
|
||||
private readonly onMessage = (channel: string, message: string): void => {
|
||||
if (channel !== this.channel(this.instanceId) || message.length > MAX_ROUTED_MESSAGE_CHARS) {
|
||||
if (
|
||||
this.destroyed ||
|
||||
channel !== this.channel(this.instanceId) ||
|
||||
message.length > MAX_ROUTED_MESSAGE_CHARS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
let parsed: unknown;
|
||||
|
|
@ -1027,6 +1085,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
}
|
||||
|
||||
private async handleRequest(request: RoutedRequest): Promise<void> {
|
||||
if (this.destroyed) return;
|
||||
if (Date.now() > request.expiresAt + REQUEST_CLOCK_SKEW_MS) {
|
||||
/** The caller stopped waiting for this long ago and has been told it was
|
||||
* unavailable, so applying it now would steer a child it believes untouched. */
|
||||
|
|
@ -1094,6 +1153,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
),
|
||||
);
|
||||
}
|
||||
if (this.destroyed) return;
|
||||
const serializedResult = JSON.stringify(result);
|
||||
/** Retaining the result rather than the envelope lets a later caller retry,
|
||||
* which carries its own correlation id, recover a response it never received. */
|
||||
|
|
@ -1102,6 +1162,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
}
|
||||
serialized = successResponse(request.requestId, serializedResult);
|
||||
} catch (error) {
|
||||
if (this.destroyed) return;
|
||||
logger.error('[subagentTaskRouting] Owner failed to process a routed command', error);
|
||||
serialized = failureResponse(request.requestId);
|
||||
}
|
||||
|
|
@ -1306,6 +1367,7 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
}
|
||||
|
||||
private async publish(channel: string, value: string): Promise<number> {
|
||||
if (this.destroyed) throw new SubagentTaskOwnerUnavailableError();
|
||||
const delivered = await this.publisher.publish(channel, value);
|
||||
return typeof delivered === 'number' ? delivered : 0;
|
||||
}
|
||||
|
|
@ -1451,5 +1513,6 @@ export class RedisSubagentTaskControlTransport implements SubagentTaskControlTra
|
|||
throw new SubagentTaskOwnerUnavailableError();
|
||||
}
|
||||
await this.ready;
|
||||
if (this.destroyed) throw new SubagentTaskOwnerUnavailableError();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,10 +25,8 @@ describe('createRedisConfigInvalidationPubSub', () => {
|
|||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const { invalidation, pubsub } of opened) {
|
||||
for (const { invalidation } of opened) {
|
||||
await invalidation.close().catch(() => undefined);
|
||||
(pubsub.publisher as Redis).disconnect();
|
||||
(pubsub.subscriber as Redis).disconnect();
|
||||
}
|
||||
for (const client of clients) {
|
||||
client.disconnect();
|
||||
|
|
@ -196,8 +194,7 @@ describe('createRedisConfigInvalidationPubSub', () => {
|
|||
server = await startRespServer();
|
||||
});
|
||||
|
||||
it('subscribes once Redis comes back when the first SUBSCRIBE failed during an outage', async () => {
|
||||
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
it('subscribes once Redis comes back when the first SUBSCRIBE is pending during an outage', async () => {
|
||||
const channel = 'librechat:config:invalidate:deploy-a';
|
||||
const client = await connect();
|
||||
const { port } = server;
|
||||
|
|
@ -205,13 +202,9 @@ describe('createRedisConfigInvalidationPubSub', () => {
|
|||
await waitFor(() => client.status !== 'ready');
|
||||
|
||||
const late = await startInstanceOn(client, 'deploy-a');
|
||||
await waitFor(
|
||||
() =>
|
||||
error.mock.calls.some(([message]) =>
|
||||
String(message).includes('Subscribing to cross-instance invalidations failed'),
|
||||
),
|
||||
5000,
|
||||
);
|
||||
const subscribe = jest.spyOn(late.pubsub.subscriber, 'subscribe');
|
||||
await waitFor(() => (late.pubsub.subscriber as Redis).status === 'reconnecting');
|
||||
expect(server.commands.filter((frame) => frame[0]?.toUpperCase() === 'SUBSCRIBE')).toEqual([]);
|
||||
|
||||
server = await startRespServer(port);
|
||||
await waitFor(subscribed(channel, 1), 5000);
|
||||
|
|
@ -222,6 +215,48 @@ describe('createRedisConfigInvalidationPubSub', () => {
|
|||
|
||||
await waitFor(() => late.clearOverrides.mock.calls.length === 1);
|
||||
expect(late.clearOverrides).toHaveBeenCalledWith('tenant-a');
|
||||
expect(subscribe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'closes its owned connections during an outage (subscription acknowledged=%s)',
|
||||
async (acknowledged) => {
|
||||
const client = await connect();
|
||||
const early = acknowledged ? await startInstanceOn(client, 'deploy-a') : undefined;
|
||||
if (early) {
|
||||
await waitFor(() => (early.pubsub.subscriber as Redis).mode === 'subscriber');
|
||||
}
|
||||
await server.close();
|
||||
await waitFor(() => client.status !== 'ready');
|
||||
const instance = early ?? (await startInstanceOn(client, 'deploy-a'));
|
||||
const subscriber = instance.pubsub.subscriber as Redis;
|
||||
const publisher = instance.pubsub.publisher as Redis;
|
||||
await waitFor(() => subscriber.status === 'reconnecting');
|
||||
const disconnectSubscriber = jest.spyOn(subscriber, 'disconnect');
|
||||
const disconnectPublisher = jest.spyOn(publisher, 'disconnect');
|
||||
const unsubscribe = jest.spyOn(subscriber, 'unsubscribe');
|
||||
|
||||
const started = Date.now();
|
||||
await instance.invalidation.close();
|
||||
await instance.invalidation.close();
|
||||
|
||||
expect(Date.now() - started).toBeLessThan(1000);
|
||||
expect(disconnectSubscriber).toHaveBeenCalledTimes(1);
|
||||
expect(disconnectPublisher).toHaveBeenCalledTimes(1);
|
||||
expect(unsubscribe).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('leaves the shared client usable when the owned pair is closed', async () => {
|
||||
const client = await connect();
|
||||
const instance = await startInstanceOn(client, 'deploy-a');
|
||||
await waitFor(() => (instance.pubsub.subscriber as Redis).mode === 'subscriber');
|
||||
|
||||
await instance.invalidation.close();
|
||||
|
||||
await expect(client.ping()).resolves.toBe('PONG');
|
||||
await waitFor(() => (instance.pubsub.subscriber as Redis).status === 'end');
|
||||
await waitFor(() => (instance.pubsub.publisher as Redis).status === 'end');
|
||||
});
|
||||
|
||||
it('returns no pub/sub without Redis', async () => {
|
||||
|
|
|
|||
|
|
@ -183,6 +183,75 @@ describe('createConfigCacheInvalidation', () => {
|
|||
expect(second.clearOverrides).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('releases borrowed listeners without waiting for an unavailable unsubscribe acknowledgement', async () => {
|
||||
const broker = createBroker();
|
||||
const subscriber = broker.subscriber();
|
||||
jest.spyOn(subscriber, 'unsubscribe').mockImplementation(() => new Promise(() => undefined));
|
||||
const invalidation = createConfigCacheInvalidation({
|
||||
clears: [],
|
||||
pubsub: { publisher: broker.publisher(), subscriber },
|
||||
});
|
||||
let closed = false;
|
||||
|
||||
void invalidation.close().then(() => {
|
||||
closed = true;
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(closed).toBe(true);
|
||||
expect(subscriber.events.listenerCount('message')).toBe(0);
|
||||
expect(subscriber.events.listenerCount('ready')).toBe(0);
|
||||
});
|
||||
|
||||
it('releases an owned pub/sub pair once instead of awaiting unsubscribe', async () => {
|
||||
const broker = createBroker();
|
||||
const subscriber = broker.subscriber();
|
||||
const unsubscribe = jest.spyOn(subscriber, 'unsubscribe');
|
||||
const close = jest.fn();
|
||||
const invalidation = createConfigCacheInvalidation({
|
||||
clears: [],
|
||||
pubsub: { publisher: broker.publisher(), subscriber, close },
|
||||
});
|
||||
await flush();
|
||||
|
||||
await invalidation.close();
|
||||
await invalidation.close();
|
||||
subscriber.events.emit('ready');
|
||||
await flush();
|
||||
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
expect(unsubscribe).not.toHaveBeenCalled();
|
||||
expect(subscriber.subscribe).toHaveBeenCalledTimes(1);
|
||||
expect(subscriber.events.listenerCount('ready')).toBe(0);
|
||||
expect(subscriber.events.listenerCount('message')).toBe(0);
|
||||
});
|
||||
|
||||
it('does not report a pending subscription rejected by intentional close as an outage', async () => {
|
||||
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
const broker = createBroker();
|
||||
const subscriber = broker.subscriber();
|
||||
let rejectSubscribe: (error: Error) => void = () => undefined;
|
||||
subscriber.subscribe.mockImplementation(
|
||||
() =>
|
||||
new Promise((_resolve, reject) => {
|
||||
rejectSubscribe = reject;
|
||||
}),
|
||||
);
|
||||
const invalidation = createConfigCacheInvalidation({
|
||||
clears: [],
|
||||
pubsub: {
|
||||
publisher: broker.publisher(),
|
||||
subscriber,
|
||||
close: () => rejectSubscribe(new Error('Connection is closed.')),
|
||||
},
|
||||
});
|
||||
|
||||
await invalidation.close();
|
||||
await flush();
|
||||
|
||||
expect(error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('subscribes again when the subscriber becomes ready after a failed SUBSCRIBE', async () => {
|
||||
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
const broker = createBroker();
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ export interface ConfigInvalidationPubSub {
|
|||
publisher: ConfigInvalidationPublisher;
|
||||
subscriber: ConfigInvalidationSubscriber;
|
||||
channel?: string;
|
||||
/** Releases owned connections without waiting for a Redis acknowledgement. */
|
||||
close?(): void;
|
||||
}
|
||||
|
||||
/** One cache a config write invalidates; `tenantId` scopes it when the cache supports that. */
|
||||
|
|
@ -158,6 +160,9 @@ export function createConfigCacheInvalidation(params: {
|
|||
},
|
||||
(error: unknown) => {
|
||||
subscribing = false;
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
logger.error(
|
||||
'[invalidateConfigCaches] Subscribing to cross-instance invalidations failed; retrying when the subscriber reconnects. Until then this instance may serve cached config until it expires:',
|
||||
getSafeErrorMetadata(error),
|
||||
|
|
@ -209,13 +214,17 @@ export function createConfigCacheInvalidation(params: {
|
|||
}
|
||||
|
||||
async function close(): Promise<void> {
|
||||
if (!pubsub) {
|
||||
if (!pubsub || closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
pubsub.subscriber.off('ready', subscribe);
|
||||
pubsub.subscriber.off('message', onMessage);
|
||||
await pubsub.subscriber.unsubscribe(channel);
|
||||
if (pubsub.close) {
|
||||
pubsub.close();
|
||||
return;
|
||||
}
|
||||
void pubsub.subscriber.unsubscribe(channel).catch(() => undefined);
|
||||
}
|
||||
|
||||
return { invalidate, reload, close };
|
||||
|
|
@ -242,9 +251,14 @@ export function createRedisConfigInvalidationPubSub(params: {
|
|||
publisher.on('error', (error: Error) => {
|
||||
logger.error('[ConfigInvalidation] publisher error:', getSafeErrorMetadata(error));
|
||||
});
|
||||
const subscriber = createIoRedisSubscriber(client, '[ConfigInvalidation] subscriber');
|
||||
return {
|
||||
publisher,
|
||||
subscriber: createIoRedisSubscriber(client, '[ConfigInvalidation] subscriber'),
|
||||
subscriber,
|
||||
channel: `${CONFIG_INVALIDATION_CHANNEL}:${namespace?.trim() || 'default'}`,
|
||||
close: () => {
|
||||
subscriber.disconnect();
|
||||
publisher.disconnect();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
217
packages/api/src/cache/__tests__/outage.cache_integration.spec.ts
vendored
Normal file
217
packages/api/src/cache/__tests__/outage.cache_integration.spec.ts
vendored
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
import type { Redis, Cluster } from 'ioredis';
|
||||
import type { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';
|
||||
import type { RedisClientsModule } from './redisClients.helper';
|
||||
import { startRedis, sleep, waitUntil } from './outage.helper';
|
||||
import { isRedisReady } from '~/app/readiness';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
jest.mock('~/utils', () => ({
|
||||
...jest.requireActual('~/utils/math'),
|
||||
...jest.requireActual('~/utils/common'),
|
||||
}));
|
||||
|
||||
describe.each([false, true])('Redis outage recovery (cluster=%s)', (cluster) => {
|
||||
let server: Awaited<ReturnType<typeof startRedis>> | undefined;
|
||||
let clients: RedisClientsModule | undefined;
|
||||
let subscriber: Redis | Cluster | undefined;
|
||||
let env: NodeJS.ProcessEnv;
|
||||
const transports: RedisEventTransport[] = [];
|
||||
const extraClients: (Redis | Cluster)[] = [];
|
||||
|
||||
const ready = (): boolean =>
|
||||
isRedisReady({
|
||||
useRedis: true,
|
||||
ioredis: clients?.ioredisClient ?? null,
|
||||
keyv: clients?.keyvRedisClient ?? null,
|
||||
});
|
||||
|
||||
const connect = async (): Promise<void> => {
|
||||
clients = await import('../redisClients');
|
||||
await clients.waitForKeyvRedisClient();
|
||||
await waitUntil(ready);
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
env = { ...process.env };
|
||||
server = await startRedis(cluster);
|
||||
Object.assign(process.env, {
|
||||
USE_REDIS: 'true',
|
||||
USE_REDIS_CLUSTER: String(cluster),
|
||||
REDIS_URI: server.url,
|
||||
REDIS_RETRY_MAX_DELAY: '100',
|
||||
REDIS_PING_INTERVAL: '0',
|
||||
REDIS_SUBSCRIBER_PING_INTERVAL: '0',
|
||||
REDIS_KEY_PREFIX: '',
|
||||
REDIS_KEY_PREFIX_VAR: '',
|
||||
REDIS_USERNAME: '',
|
||||
REDIS_PASSWORD: '',
|
||||
REDIS_CA: '',
|
||||
REDIS_ENABLE_OFFLINE_QUEUE: 'true',
|
||||
});
|
||||
delete process.env.REDIS_RETRY_MAX_ATTEMPTS;
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const transport of transports.splice(0)) {
|
||||
transport.destroy();
|
||||
}
|
||||
for (const client of extraClients.splice(0)) {
|
||||
client.disconnect();
|
||||
}
|
||||
subscriber?.disconnect();
|
||||
subscriber = undefined;
|
||||
clients?.ioredisClient?.disconnect();
|
||||
const keyv = clients?.keyvRedisClient;
|
||||
try {
|
||||
if (keyv?.isOpen) {
|
||||
keyv.destroy();
|
||||
}
|
||||
} catch (error) {
|
||||
// Cluster teardown can encounter a master that exhausted its finite budget.
|
||||
if (!(error instanceof Error) || error.message !== 'The client is closed') {
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
clients = undefined;
|
||||
try {
|
||||
await server?.close();
|
||||
} finally {
|
||||
server = undefined;
|
||||
process.env = env;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers both clients and subscriptions after 20 seconds without replaying an uncertain write', async () => {
|
||||
await connect();
|
||||
const io = clients!.ioredisClient!;
|
||||
const keyv = clients!.keyvRedisClient!;
|
||||
const { createIoRedisSubscriber } = await import('../redisUtils');
|
||||
subscriber = createIoRedisSubscriber(io, 'outage-subscriber');
|
||||
const messages: string[] = [];
|
||||
subscriber.on('message', (_channel: string, value: string) => messages.push(value));
|
||||
await waitUntil(() => subscriber?.status === 'ready');
|
||||
await subscriber.subscribe('outage-events');
|
||||
await io.publish('outage-events', 'before');
|
||||
await waitUntil(() => messages.includes('before'));
|
||||
|
||||
server!.dropReplies();
|
||||
const pendingWrite = io.incr('outage-counter').then(
|
||||
() => 'resolved',
|
||||
() => 'rejected',
|
||||
);
|
||||
const pendingKeyvWrite = keyv.incr('keyv-outage-counter').then(
|
||||
() => 'resolved',
|
||||
() => 'rejected',
|
||||
);
|
||||
let applied = false;
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!applied && Date.now() < deadline) {
|
||||
applied =
|
||||
(await server!.admin.get('::outage-counter')) === '1' &&
|
||||
(await server!.admin.get('keyv-outage-counter')) === '1';
|
||||
if (!applied) await sleep(20);
|
||||
}
|
||||
expect(applied).toBe(true);
|
||||
server!.block();
|
||||
await waitUntil(() => !ready());
|
||||
await sleep(20_000);
|
||||
expect(ready()).toBe(false);
|
||||
await expect(pendingWrite).resolves.toBe('rejected');
|
||||
await expect(pendingKeyvWrite).resolves.toBe('rejected');
|
||||
|
||||
server!.restore();
|
||||
await waitUntil(ready);
|
||||
await waitUntil(() => subscriber?.status === 'ready');
|
||||
await expect(io.set('after-outage', 'ioredis')).resolves.toBe('OK');
|
||||
await expect(io.get('after-outage')).resolves.toBe('ioredis');
|
||||
await expect(keyv.set('keyv-after-outage', 'keyv')).resolves.toBe('OK');
|
||||
await expect(keyv.get('keyv-after-outage')).resolves.toBe('keyv');
|
||||
await expect(server!.admin.get('::outage-counter')).resolves.toBe('1');
|
||||
await expect(server!.admin.get('keyv-outage-counter')).resolves.toBe('1');
|
||||
await io.publish('outage-events', 'after');
|
||||
await waitUntil(() => messages.includes('after'));
|
||||
expect(messages).toEqual(['before', 'after']);
|
||||
}, 40_000);
|
||||
|
||||
it('keeps stream and abort subscriptions after an interrupted resubscribe acknowledgement', async () => {
|
||||
await connect();
|
||||
const { createIoRedisSubscriber, duplicateIoRedisClient } = await import('../redisUtils');
|
||||
const { RedisEventTransport } = await import('~/stream/implementations/RedisEventTransport');
|
||||
const io = clients!.ioredisClient!;
|
||||
subscriber = createIoRedisSubscriber(io, 'outage-owner');
|
||||
let readyEvents = 0;
|
||||
subscriber.on('ready', () => readyEvents++);
|
||||
const publisher = duplicateIoRedisClient(io);
|
||||
extraClients.push(publisher);
|
||||
const owner = new RedisEventTransport(io, subscriber);
|
||||
const peer = new RedisEventTransport(publisher, createIoRedisSubscriber(io, 'outage-peer'));
|
||||
transports.push(owner, peer);
|
||||
const streamId = 'interrupted-subscription';
|
||||
const channel = `stream:{${streamId}}:events`;
|
||||
const chunks: unknown[] = [];
|
||||
let abortedGeneration: number | undefined;
|
||||
const attachment = owner.subscribe(streamId, { onChunk: (event) => chunks.push(event) });
|
||||
await attachment.ready;
|
||||
const unsubscribeAbort = await owner.onAbort(streamId, (generation) => {
|
||||
abortedGeneration = generation;
|
||||
});
|
||||
await waitUntil(() => publisher.status === 'ready');
|
||||
await peer.emitChunk(streamId, { value: 'before' });
|
||||
await waitUntil(() => chunks.length === 1);
|
||||
|
||||
server!.block();
|
||||
await waitUntil(() => !ready());
|
||||
server!.interruptNextSubscription();
|
||||
server!.restore();
|
||||
await waitUntil(() => server!.interruptedSubscriptions === 1);
|
||||
await waitUntil(
|
||||
() => ready() && subscriber?.status === 'ready' && readyEvents >= (cluster ? 2 : 3),
|
||||
);
|
||||
|
||||
const isSubscribed = async (): Promise<boolean> => {
|
||||
const counts = await server!.admin.pubsub('NUMSUB', channel);
|
||||
return Array.isArray(counts) && counts[1] === 1;
|
||||
};
|
||||
let subscribed = await isSubscribed();
|
||||
const deadline = Date.now() + 2000;
|
||||
while (!subscribed && Date.now() < deadline) {
|
||||
await sleep(20);
|
||||
subscribed = await isSubscribed();
|
||||
}
|
||||
expect(subscribed).toBe(true);
|
||||
await peer.emitChunk(streamId, { value: 'after' });
|
||||
peer.emitAbort(streamId, 7);
|
||||
await waitUntil(() => chunks.length === 2 && abortedGeneration === 7);
|
||||
expect(chunks).toEqual([{ value: 'before' }, { value: 'after' }]);
|
||||
unsubscribeAbort();
|
||||
attachment.unsubscribe();
|
||||
}, 15_000);
|
||||
|
||||
it('honors an explicit finite reconnect budget and remains not ready', async () => {
|
||||
process.env.REDIS_RETRY_MAX_ATTEMPTS = '2';
|
||||
await connect();
|
||||
const { createIoRedisSubscriber } = await import('../redisUtils');
|
||||
subscriber = createIoRedisSubscriber(clients!.ioredisClient!, 'finite-subscriber');
|
||||
await waitUntil(() => subscriber?.status === 'ready');
|
||||
await subscriber.subscribe('finite-events');
|
||||
server!.block();
|
||||
await waitUntil(() => !ready());
|
||||
await sleep(2000);
|
||||
server!.restore();
|
||||
await sleep(500);
|
||||
expect(ready()).toBe(false);
|
||||
expect(clients!.ioredisClient!.status).not.toBe('ready');
|
||||
expect(subscriber.status).not.toBe('ready');
|
||||
expect(
|
||||
isRedisReady({
|
||||
useRedis: true,
|
||||
ioredis: { status: 'ready' },
|
||||
keyv: clients!.keyvRedisClient,
|
||||
}),
|
||||
).toBe(false);
|
||||
}, 15_000);
|
||||
});
|
||||
220
packages/api/src/cache/__tests__/outage.helper.ts
vendored
Normal file
220
packages/api/src/cache/__tests__/outage.helper.ts
vendored
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
import net from 'node:net';
|
||||
import Redis from 'ioredis';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { once } from 'node:events';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import type { ChildProcess } from 'node:child_process';
|
||||
import type { Socket } from 'node:net';
|
||||
|
||||
export const sleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export async function waitUntil(predicate: () => boolean, timeout = 5000): Promise<void> {
|
||||
const deadline = Date.now() + timeout;
|
||||
while (!predicate()) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error('Timed out waiting for Redis state');
|
||||
}
|
||||
await sleep(20);
|
||||
}
|
||||
}
|
||||
|
||||
async function freePort(): Promise<number> {
|
||||
const server = net.createServer();
|
||||
try {
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Expected a TCP address');
|
||||
}
|
||||
return address.port;
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
/** Real Redis behind a TCP fault boundary, including the port advertised to Cluster clients. */
|
||||
export async function startRedis(cluster: boolean) {
|
||||
const port = await freePort();
|
||||
let busPort = await freePort();
|
||||
while (busPort === port) {
|
||||
busPort = await freePort();
|
||||
}
|
||||
const sockets = new Set<Socket>();
|
||||
let blocked = false;
|
||||
let dropReplies = false;
|
||||
let interruptSubscription = false;
|
||||
let interruptedSubscriptions = 0;
|
||||
const proxy = net.createServer((socket) => {
|
||||
if (blocked) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
const upstream = net.connect(port, '127.0.0.1');
|
||||
for (const connection of [socket, upstream]) {
|
||||
sockets.add(connection);
|
||||
connection.on('error', () => connection.destroy());
|
||||
connection.on('close', () => {
|
||||
sockets.delete(connection);
|
||||
socket.destroy();
|
||||
upstream.destroy();
|
||||
});
|
||||
}
|
||||
socket.pipe(upstream);
|
||||
let replyTail = '';
|
||||
const subscribeReply = '$9\r\nsubscribe\r\n';
|
||||
upstream.on('data', (chunk: Buffer) => {
|
||||
const reply = replyTail + chunk.toString('latin1');
|
||||
replyTail = reply.slice(-subscribeReply.length);
|
||||
if (interruptSubscription && reply.includes(subscribeReply)) {
|
||||
interruptSubscription = false;
|
||||
interruptedSubscriptions++;
|
||||
socket.destroy();
|
||||
upstream.destroy();
|
||||
return;
|
||||
}
|
||||
if (!dropReplies) {
|
||||
socket.write(chunk);
|
||||
}
|
||||
});
|
||||
});
|
||||
let dir: string | undefined;
|
||||
let child: ChildProcess | undefined;
|
||||
let childClosed: Promise<void> | undefined;
|
||||
let admin: Redis | undefined;
|
||||
let startupError: Error | undefined;
|
||||
const close = async (): Promise<void> => {
|
||||
admin?.disconnect();
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
const proxyClosed = new Promise<void>((resolve) => proxy.close(() => resolve()));
|
||||
if (child?.pid && child.exitCode === null && child.signalCode === null) {
|
||||
child.kill('SIGTERM');
|
||||
}
|
||||
await Promise.all([proxyClosed, childClosed]);
|
||||
if (dir) {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
proxy.listen(0, '127.0.0.1');
|
||||
await once(proxy, 'listening');
|
||||
const address = proxy.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Expected a TCP proxy address');
|
||||
}
|
||||
dir = await mkdtemp(join(tmpdir(), 'redis-outage-'));
|
||||
const binary = process.env.REDIS_SERVER_BIN ?? 'redis-server';
|
||||
child = spawn(
|
||||
binary,
|
||||
[
|
||||
'--bind',
|
||||
'127.0.0.1',
|
||||
'--port',
|
||||
String(port),
|
||||
'--dir',
|
||||
dir,
|
||||
'--save',
|
||||
'',
|
||||
'--appendonly',
|
||||
'no',
|
||||
'--unixsocket',
|
||||
'',
|
||||
...(cluster
|
||||
? [
|
||||
'--cluster-enabled',
|
||||
'yes',
|
||||
'--cluster-config-file',
|
||||
'nodes.conf',
|
||||
'--cluster-port',
|
||||
String(busPort),
|
||||
'--cluster-announce-ip',
|
||||
'127.0.0.1',
|
||||
'--cluster-announce-port',
|
||||
String(address.port),
|
||||
'--cluster-announce-bus-port',
|
||||
String(busPort),
|
||||
]
|
||||
: []),
|
||||
],
|
||||
{ stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
);
|
||||
let stderr = '';
|
||||
child.stdout?.resume();
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
stderr = (stderr + chunk.toString()).slice(-4096);
|
||||
});
|
||||
child.on('error', (error: Error) => {
|
||||
startupError = new Error(`Unable to start Redis (${binary}): ${error.message}`, {
|
||||
cause: error,
|
||||
});
|
||||
});
|
||||
childClosed = new Promise<void>((resolve) => {
|
||||
child!.once('close', (code, signal) => {
|
||||
startupError ??= new Error(
|
||||
`Redis exited before becoming ready (code=${code}, signal=${signal}): ${stderr.trim()}`,
|
||||
);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
admin = new Redis(port, '127.0.0.1', {
|
||||
retryStrategy: () => 20,
|
||||
enableReadyCheck: false,
|
||||
maxRetriesPerRequest: 3,
|
||||
keyPrefix: '',
|
||||
});
|
||||
admin.on('error', () => undefined);
|
||||
await waitUntil(() => {
|
||||
if (startupError) {
|
||||
throw startupError;
|
||||
}
|
||||
return admin?.status === 'ready';
|
||||
});
|
||||
if (cluster) {
|
||||
await admin.cluster('ADDSLOTSRANGE', '0', '16383');
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!((await admin.cluster('INFO')) as string).includes('cluster_state:ok')) {
|
||||
if (startupError) {
|
||||
throw startupError;
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error('Redis cluster did not become ready');
|
||||
}
|
||||
await sleep(50);
|
||||
}
|
||||
}
|
||||
return {
|
||||
url: `redis://127.0.0.1:${address.port}`,
|
||||
admin,
|
||||
dropReplies: () => {
|
||||
dropReplies = true;
|
||||
},
|
||||
interruptNextSubscription: () => {
|
||||
interruptSubscription = true;
|
||||
},
|
||||
get interruptedSubscriptions() {
|
||||
return interruptedSubscriptions;
|
||||
},
|
||||
block: () => {
|
||||
blocked = true;
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
},
|
||||
restore: () => {
|
||||
blocked = false;
|
||||
dropReplies = false;
|
||||
},
|
||||
close,
|
||||
};
|
||||
} catch (error) {
|
||||
const failure = startupError ?? error;
|
||||
await close();
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
97
packages/api/src/cache/__tests__/outage.spec.ts
vendored
Normal file
97
packages/api/src/cache/__tests__/outage.spec.ts
vendored
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import net from 'node:net';
|
||||
import Redis from 'ioredis';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import fs from 'node:fs/promises';
|
||||
import childProcess from 'node:child_process';
|
||||
import { startRedis } from './outage.helper';
|
||||
|
||||
describe.each([false, true])('Redis outage fixture startup (cluster=%s)', (cluster) => {
|
||||
let env: NodeJS.ProcessEnv;
|
||||
let dir: string;
|
||||
let servers: jest.SpiedFunction<typeof net.createServer>;
|
||||
let directories: jest.SpiedFunction<typeof fs.mkdtemp>;
|
||||
let disconnect: jest.SpiedFunction<typeof Redis.prototype.disconnect>;
|
||||
let spawned: jest.SpiedFunction<typeof childProcess.spawn>;
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await fs.mkdtemp(join(tmpdir(), 'redis-fixture-test-'));
|
||||
await fs.writeFile(
|
||||
join(dir, 'exits'),
|
||||
`#!${process.execPath}\nprocess.stderr.write('fixture startup refused\\n', () => process.exit(23));\n`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
await fs.writeFile(
|
||||
join(dir, 'idle'),
|
||||
`#!${process.execPath}\nsetInterval(() => undefined, 1000);\n`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
env = { ...process.env };
|
||||
servers = jest.spyOn(net, 'createServer');
|
||||
directories = jest.spyOn(fs, 'mkdtemp');
|
||||
disconnect = jest.spyOn(Redis.prototype, 'disconnect');
|
||||
spawned = jest.spyOn(childProcess, 'spawn');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = env;
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const expectServersClosed = (): void => {
|
||||
expect(servers).toHaveBeenCalledTimes(3);
|
||||
for (const result of servers.mock.results) {
|
||||
if (result.type === 'return') {
|
||||
expect(result.value.listening).toBe(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const expectCleanup = async (): Promise<void> => {
|
||||
expectServersClosed();
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
expect(directories).toHaveBeenCalledTimes(1);
|
||||
const directory: string = await directories.mock.results[0].value;
|
||||
await expect(fs.access(directory)).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
};
|
||||
|
||||
it('rejects a missing binary promptly and closes its proxy, admin, and directory', async () => {
|
||||
const binary = join(dir, 'missing-redis-server');
|
||||
process.env.REDIS_SERVER_BIN = binary;
|
||||
await expect(startRedis(cluster)).rejects.toThrow(
|
||||
`Unable to start Redis (${binary}): spawn ${binary} ENOENT`,
|
||||
);
|
||||
await expectCleanup();
|
||||
}, 3000);
|
||||
|
||||
it('reports an early exit with its code and stderr, then cleans up', async () => {
|
||||
process.env.REDIS_SERVER_BIN = join(dir, 'exits');
|
||||
await expect(startRedis(cluster)).rejects.toThrow(
|
||||
'Redis exited before becoming ready (code=23, signal=null): fixture startup refused',
|
||||
);
|
||||
await expectCleanup();
|
||||
}, 3000);
|
||||
|
||||
it('terminates a live child and removes its resources when readiness times out', async () => {
|
||||
process.env.REDIS_SERVER_BIN = join(dir, 'idle');
|
||||
await expect(startRedis(cluster)).rejects.toThrow('Timed out waiting for Redis state');
|
||||
expect(spawned).toHaveBeenCalledTimes(1);
|
||||
expect(spawned.mock.results[0].value.signalCode).toBe('SIGTERM');
|
||||
await expectCleanup();
|
||||
});
|
||||
|
||||
it('closes the proxy if allocating the Redis directory fails', async () => {
|
||||
const error = new Error('fixture directory unavailable');
|
||||
directories.mockRejectedValueOnce(error);
|
||||
await expect(startRedis(cluster)).rejects.toBe(error);
|
||||
expectServersClosed();
|
||||
expect(disconnect).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
33
packages/api/src/cache/cacheConfig.spec.ts
vendored
Normal file
33
packages/api/src/cache/cacheConfig.spec.ts
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
jest.mock('@librechat/data-schemas', () => ({ logger: { warn: jest.fn(), error: jest.fn() } }));
|
||||
jest.mock('~/utils', () => ({
|
||||
...jest.requireActual('~/utils/math'),
|
||||
...jest.requireActual('~/utils/common'),
|
||||
}));
|
||||
|
||||
describe('Redis retry configuration', () => {
|
||||
const original = process.env.REDIS_RETRY_MAX_ATTEMPTS;
|
||||
|
||||
afterEach(() => {
|
||||
if (original === undefined) {
|
||||
delete process.env.REDIS_RETRY_MAX_ATTEMPTS;
|
||||
} else {
|
||||
process.env.REDIS_RETRY_MAX_ATTEMPTS = original;
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, 0],
|
||||
['0', 0],
|
||||
['5', 5],
|
||||
])('reads %s as %s attempts', (value, expected) => {
|
||||
if (value === undefined) {
|
||||
delete process.env.REDIS_RETRY_MAX_ATTEMPTS;
|
||||
} else {
|
||||
process.env.REDIS_RETRY_MAX_ATTEMPTS = value;
|
||||
}
|
||||
jest.isolateModules(() => {
|
||||
const { cacheConfig } = jest.requireActual<typeof import('./cacheConfig')>('./cacheConfig');
|
||||
expect(cacheConfig.REDIS_RETRY_MAX_ATTEMPTS).toBe(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
2
packages/api/src/cache/cacheConfig.ts
vendored
2
packages/api/src/cache/cacheConfig.ts
vendored
|
|
@ -174,7 +174,7 @@ const cacheConfig: {
|
|||
/** Max delay between reconnection attempts in ms */
|
||||
REDIS_RETRY_MAX_DELAY: math(process.env.REDIS_RETRY_MAX_DELAY, 3000),
|
||||
/** Max number of reconnection attempts (0 = infinite) */
|
||||
REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 10),
|
||||
REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 0),
|
||||
/** Connection timeout in ms */
|
||||
REDIS_CONNECT_TIMEOUT: math(process.env.REDIS_CONNECT_TIMEOUT, 10000),
|
||||
/** Min spacing in ms between reconnects forced by READONLY replies after a failover */
|
||||
|
|
|
|||
95
packages/api/src/cache/redisClients.ts
vendored
95
packages/api/src/cache/redisClients.ts
vendored
|
|
@ -7,6 +7,7 @@ import type { RedisClientType, RedisClusterType } from '@redis/client';
|
|||
import type { Redis, Cluster } from 'ioredis';
|
||||
import type { ReadonlyRecoveryHandler } from './recovery';
|
||||
import { createReadonlyRecovery, isReadonlyReplicaError } from './recovery';
|
||||
import { createRedisRetryStrategy } from './retry';
|
||||
import { startRedisHeartbeat } from './heartbeat';
|
||||
import { cacheConfig } from './cacheConfig';
|
||||
|
||||
|
|
@ -48,22 +49,13 @@ if (cacheConfig.USE_REDIS) {
|
|||
tls: useTls ? { ca: ca ?? undefined } : undefined,
|
||||
keyPrefix: `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR}`,
|
||||
maxListeners: cacheConfig.REDIS_MAX_LISTENERS,
|
||||
retryStrategy: (times: number) => {
|
||||
if (
|
||||
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
|
||||
times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
|
||||
) {
|
||||
logger.error(
|
||||
`ioredis giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const base = Math.min(Math.pow(2, times) * 50, cacheConfig.REDIS_RETRY_MAX_DELAY);
|
||||
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
|
||||
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
|
||||
logger.info(`ioredis reconnecting... attempt ${times}, delay ${delay}ms`);
|
||||
return delay;
|
||||
},
|
||||
retryStrategy: createRedisRetryStrategy({
|
||||
label: 'ioredis',
|
||||
maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS,
|
||||
maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY,
|
||||
initialDelay: 50,
|
||||
log: (message) => logger.error(message),
|
||||
}),
|
||||
reconnectOnError: (err: Error) => {
|
||||
if (isReadonlyReplicaError(err)) {
|
||||
logger.warn('ioredis reconnecting due to READONLY error');
|
||||
|
|
@ -73,7 +65,9 @@ if (cacheConfig.USE_REDIS) {
|
|||
},
|
||||
enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
|
||||
connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
|
||||
maxRetriesPerRequest: 3,
|
||||
// A lost reply does not prove a write failed. Reconnect without replaying it.
|
||||
maxRetriesPerRequest: 0,
|
||||
autoResendUnfulfilledCommands: false,
|
||||
keepAlive: cacheConfig.REDIS_KEEP_ALIVE,
|
||||
};
|
||||
|
||||
|
|
@ -91,32 +85,25 @@ if (cacheConfig.USE_REDIS) {
|
|||
}
|
||||
: {}),
|
||||
redisOptions,
|
||||
clusterRetryStrategy: (times: number) => {
|
||||
if (
|
||||
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
|
||||
times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
|
||||
) {
|
||||
logger.error(
|
||||
`ioredis cluster giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const base = Math.min(Math.pow(2, times) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
|
||||
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
|
||||
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
|
||||
logger.info(`ioredis cluster reconnecting... attempt ${times}, delay ${delay}ms`);
|
||||
return delay;
|
||||
},
|
||||
// Keep MOVED/ASK routing, but never resend a command after losing its reply.
|
||||
retryDelayOnFailover: 0,
|
||||
clusterRetryStrategy: createRedisRetryStrategy({
|
||||
label: 'ioredis cluster',
|
||||
maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS,
|
||||
maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY,
|
||||
initialDelay: 100,
|
||||
log: (message) => logger.error(message),
|
||||
}),
|
||||
enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
|
||||
},
|
||||
);
|
||||
|
||||
ioredisClient.on('error', (err) => {
|
||||
logger.error('ioredis client error:', err);
|
||||
logger.debug('ioredis client error:', err);
|
||||
});
|
||||
|
||||
ioredisClient.on('connect', () => {
|
||||
logger.info('ioredis client connected');
|
||||
logger.debug('ioredis client connected');
|
||||
});
|
||||
|
||||
ioredisClient.on('ready', () => {
|
||||
|
|
@ -124,11 +111,11 @@ if (cacheConfig.USE_REDIS) {
|
|||
});
|
||||
|
||||
ioredisClient.on('reconnecting', (delay: number) => {
|
||||
logger.info(`ioredis client reconnecting in ${delay}ms`);
|
||||
logger.debug(`ioredis client reconnecting in ${delay}ms`);
|
||||
});
|
||||
|
||||
ioredisClient.on('close', () => {
|
||||
logger.warn('ioredis client connection closed');
|
||||
logger.debug('ioredis client connection closed');
|
||||
});
|
||||
|
||||
/** Deadline-bounded keepalive PINGs; an unanswered probe forces a reconnect. */
|
||||
|
|
@ -201,6 +188,14 @@ if (cacheConfig.USE_REDIS) {
|
|||
* ** WARNING ** Keyv Redis client does not support Prefix like ioredis above.
|
||||
* The prefix feature will be handled by the Keyv-Redis store in cacheFactory.js
|
||||
*/
|
||||
const retry = createRedisRetryStrategy({
|
||||
label: '@keyv/redis client',
|
||||
maxAttempts: cacheConfig.REDIS_RETRY_MAX_ATTEMPTS,
|
||||
maxDelay: cacheConfig.REDIS_RETRY_MAX_DELAY,
|
||||
initialDelay: 100,
|
||||
firstAttempt: 0,
|
||||
log: (message) => logger.error(message),
|
||||
});
|
||||
const redisOptions: Record<string, unknown> = {
|
||||
username,
|
||||
password,
|
||||
|
|
@ -208,22 +203,8 @@ if (cacheConfig.USE_REDIS) {
|
|||
...(isRedisCluster ? { tls: useTls } : {}),
|
||||
...(ca ? { ca } : {}),
|
||||
connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
|
||||
reconnectStrategy: (retries: number) => {
|
||||
if (
|
||||
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
|
||||
retries > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
|
||||
) {
|
||||
logger.error(
|
||||
`@keyv/redis client giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
|
||||
);
|
||||
return new Error('Max reconnection attempts reached');
|
||||
}
|
||||
const base = Math.min(Math.pow(2, retries) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
|
||||
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
|
||||
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
|
||||
logger.info(`@keyv/redis reconnecting... attempt ${retries}, delay ${delay}ms`);
|
||||
return delay;
|
||||
},
|
||||
reconnectStrategy: (retries: number) =>
|
||||
retry(retries) ?? new Error('Max reconnection attempts reached'),
|
||||
},
|
||||
disableOfflineQueue: !cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
|
||||
...(cacheConfig.REDIS_PING_INTERVAL > 0
|
||||
|
|
@ -264,11 +245,11 @@ if (cacheConfig.USE_REDIS) {
|
|||
}
|
||||
|
||||
keyvRedisClient.on('error', (err) => {
|
||||
logger.error('@keyv/redis client error:', err);
|
||||
logger.debug('@keyv/redis client error:', err);
|
||||
});
|
||||
|
||||
keyvRedisClient.on('connect', () => {
|
||||
logger.info('@keyv/redis client connected');
|
||||
logger.debug('@keyv/redis client connected');
|
||||
});
|
||||
|
||||
keyvRedisClient.on('ready', () => {
|
||||
|
|
@ -276,11 +257,11 @@ if (cacheConfig.USE_REDIS) {
|
|||
});
|
||||
|
||||
keyvRedisClient.on('reconnecting', () => {
|
||||
logger.info('@keyv/redis client reconnecting...');
|
||||
logger.debug('@keyv/redis client reconnecting...');
|
||||
});
|
||||
|
||||
keyvRedisClient.on('disconnect', () => {
|
||||
logger.warn('@keyv/redis client disconnected');
|
||||
logger.debug('@keyv/redis client disconnected');
|
||||
});
|
||||
|
||||
// Start connection immediately and settle the gate created before client initialization.
|
||||
|
|
|
|||
37
packages/api/src/cache/redisUtils.spec.ts
vendored
37
packages/api/src/cache/redisUtils.spec.ts
vendored
|
|
@ -1,6 +1,7 @@
|
|||
import IoRedis from 'ioredis';
|
||||
import type { RedisClientType } from '@redis/client';
|
||||
import { duplicateIoRedisClient, scanKeys } from './redisUtils';
|
||||
import type { Redis, Cluster } from 'ioredis';
|
||||
import { createIoRedisSubscriber, duplicateIoRedisClient, scanKeys } from './redisUtils';
|
||||
|
||||
describe('duplicateIoRedisClient', () => {
|
||||
it('applies overrides to a single-node duplicate', () => {
|
||||
|
|
@ -81,6 +82,40 @@ describe('duplicateIoRedisClient', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe.each([false, true])('createIoRedisSubscriber (cluster=%s)', (cluster) => {
|
||||
it('retains subscriber commands without changing the publisher retry policy or options', () => {
|
||||
const redisOptions = {
|
||||
maxRetriesPerRequest: 0,
|
||||
autoResendUnfulfilledCommands: false,
|
||||
keyPrefix: 'subscriber-test::',
|
||||
};
|
||||
const publisher = cluster
|
||||
? new IoRedis.Cluster([{ host: '127.0.0.1', port: 6379 }], {
|
||||
lazyConnect: true,
|
||||
redisOptions,
|
||||
})
|
||||
: new IoRedis({ ...redisOptions, lazyConnect: true });
|
||||
const subscriber = createIoRedisSubscriber(publisher, 'test-subscriber');
|
||||
const options = subscriber.isCluster
|
||||
? (subscriber as Cluster).options.redisOptions
|
||||
: (subscriber as Redis).options;
|
||||
const publisherOptions = publisher.isCluster
|
||||
? (publisher as Cluster).options.redisOptions
|
||||
: (publisher as Redis).options;
|
||||
try {
|
||||
expect(options).toMatchObject({
|
||||
maxRetriesPerRequest: null,
|
||||
autoResendUnfulfilledCommands: true,
|
||||
keyPrefix: redisOptions.keyPrefix,
|
||||
});
|
||||
expect(publisherOptions).toMatchObject(redisOptions);
|
||||
} finally {
|
||||
subscriber.disconnect();
|
||||
publisher.disconnect();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('scanKeys', () => {
|
||||
it('flattens node-redis v5 scan pages', async () => {
|
||||
const client = {
|
||||
|
|
|
|||
21
packages/api/src/cache/redisUtils.ts
vendored
21
packages/api/src/cache/redisUtils.ts
vendored
|
|
@ -53,19 +53,36 @@ export function duplicateIoRedisClient(
|
|||
* The heartbeat is the traffic the shared client gets for free.
|
||||
*/
|
||||
export function createIoRedisSubscriber(client: Redis | Cluster, label: string): Redis | Cluster {
|
||||
const subscriber = duplicateIoRedisClient(client);
|
||||
// Retain idempotent SUBSCRIBE/PING commands across lost acknowledgements. Publishers
|
||||
// keep their fail-fast policy so uncertain application writes are never replayed.
|
||||
const redisOptions: RedisOptions = {
|
||||
maxRetriesPerRequest: null,
|
||||
autoResendUnfulfilledCommands: true,
|
||||
};
|
||||
const subscriber = duplicateIoRedisClient(
|
||||
client,
|
||||
client.isCluster
|
||||
? { redisOptions: { ...(client as Cluster).options.redisOptions, ...redisOptions } }
|
||||
: redisOptions,
|
||||
);
|
||||
subscriber.on('error', (error: Error) => {
|
||||
logger.error(`${label} error:`, error);
|
||||
});
|
||||
subscriber.on('ready', () => {
|
||||
logger.info(`${label} ready`);
|
||||
});
|
||||
startRedisHeartbeat({
|
||||
const stopHeartbeat = startRedisHeartbeat({
|
||||
client: subscriber,
|
||||
intervalMs: cacheConfig.REDIS_SUBSCRIBER_PING_INTERVAL * 1000,
|
||||
timeoutMs: cacheConfig.REDIS_PING_TIMEOUT,
|
||||
label,
|
||||
});
|
||||
// An already-closed reconnecting socket need not emit another `end`.
|
||||
const disconnect = subscriber.disconnect.bind(subscriber);
|
||||
subscriber.disconnect = (reconnect?: boolean): void => {
|
||||
if (reconnect !== true) stopHeartbeat();
|
||||
disconnect(reconnect);
|
||||
};
|
||||
return subscriber;
|
||||
}
|
||||
|
||||
|
|
|
|||
66
packages/api/src/cache/retry.spec.ts
vendored
Normal file
66
packages/api/src/cache/retry.spec.ts
vendored
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import { createRedisRetryStrategy } from './retry';
|
||||
|
||||
describe('Redis reconnect policy', () => {
|
||||
const log = jest.fn();
|
||||
const options = {
|
||||
label: 'Redis',
|
||||
maxAttempts: 0,
|
||||
maxDelay: 3000,
|
||||
initialDelay: 50,
|
||||
log,
|
||||
};
|
||||
|
||||
beforeEach(() => log.mockClear());
|
||||
|
||||
it('keeps retrying for the lifetime of the process without overflowing the backoff', () => {
|
||||
const retry = createRedisRetryStrategy(options);
|
||||
for (const attempt of [1, 10, 100, 1024, 100_000, Number.MAX_SAFE_INTEGER]) {
|
||||
const delay = retry(attempt);
|
||||
expect(Number.isFinite(delay)).toBe(true);
|
||||
expect(delay).toBeGreaterThan(0);
|
||||
expect(delay).toBeLessThanOrEqual(options.maxDelay);
|
||||
}
|
||||
});
|
||||
|
||||
it('retains jitter once the exponential delay has reached its cap', () => {
|
||||
const random = jest.spyOn(Math, 'random');
|
||||
const retry = createRedisRetryStrategy(options);
|
||||
random.mockReturnValueOnce(0).mockReturnValueOnce(0.999);
|
||||
expect(retry(10_000)).toBe(3000);
|
||||
expect(retry(10_000)).toBe(2001);
|
||||
});
|
||||
|
||||
it.each([50, 100])(
|
||||
'grows the existing %sms backoff and respects a smaller cap',
|
||||
(initialDelay) => {
|
||||
jest.spyOn(Math, 'random').mockReturnValue(0);
|
||||
const retry = createRedisRetryStrategy({ ...options, initialDelay });
|
||||
expect(retry(1)).toBe(initialDelay * 2);
|
||||
expect(retry(2)).toBe(initialDelay * 4);
|
||||
expect(createRedisRetryStrategy({ ...options, initialDelay, maxDelay: 1 })(100_000)).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([0, 1])('preserves finite overrides for a client counting from %s', (firstAttempt) => {
|
||||
const retry = createRedisRetryStrategy({ ...options, maxAttempts: 2, firstAttempt });
|
||||
for (let attempt = firstAttempt; attempt <= 2; attempt++) {
|
||||
expect(retry(attempt)).not.toBeNull();
|
||||
}
|
||||
expect(retry(3)).toBeNull();
|
||||
expect(log).toHaveBeenLastCalledWith('Redis giving up after 2 reconnection attempts');
|
||||
});
|
||||
|
||||
it.each([0, 1])(
|
||||
'logs the first and every tenth failure for a client counting from %s',
|
||||
(firstAttempt) => {
|
||||
const retry = createRedisRetryStrategy({ ...options, firstAttempt });
|
||||
for (let attempt = firstAttempt; attempt < firstAttempt + 100; attempt++) {
|
||||
retry(attempt);
|
||||
}
|
||||
expect(log).toHaveBeenCalledTimes(11);
|
||||
expect(log.mock.calls[0][0]).toContain('attempt 1,');
|
||||
expect(log.mock.calls[1][0]).toContain('attempt 10,');
|
||||
expect(log.mock.calls[10][0]).toContain('attempt 100,');
|
||||
},
|
||||
);
|
||||
});
|
||||
36
packages/api/src/cache/retry.ts
vendored
Normal file
36
packages/api/src/cache/retry.ts
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
type RedisRetryOptions = {
|
||||
label: string;
|
||||
maxAttempts: number;
|
||||
maxDelay: number;
|
||||
initialDelay: number;
|
||||
firstAttempt?: number;
|
||||
log: (message: string) => void;
|
||||
};
|
||||
|
||||
/** Reconnect transport only; callers still own failed commands and their side effects. */
|
||||
export function createRedisRetryStrategy({
|
||||
label,
|
||||
maxAttempts,
|
||||
maxDelay,
|
||||
initialDelay,
|
||||
firstAttempt = 1,
|
||||
log,
|
||||
}: RedisRetryOptions): (attempt: number) => number | null {
|
||||
const maxExponent = Math.max(0, Math.ceil(Math.log2(maxDelay / initialDelay)));
|
||||
return (attempt) => {
|
||||
if (maxAttempts > 0 && attempt > maxAttempts) {
|
||||
log(`${label} giving up after ${maxAttempts} reconnection attempts`);
|
||||
return null;
|
||||
}
|
||||
// Saturate the exponent before computing it, even after months of reconnects.
|
||||
const exponent = Math.min(attempt, maxExponent);
|
||||
const base = Math.min(initialDelay * 2 ** exponent, maxDelay);
|
||||
// Keep jitter at the cap rather than clipping every retry to the same delay.
|
||||
const delay = base - Math.floor(Math.random() * Math.min(base / 2, 1000));
|
||||
const failure = attempt - firstAttempt + 1;
|
||||
if (failure === 1 || failure % 10 === 0) {
|
||||
log(`${label} reconnecting... attempt ${failure}, delay ${delay}ms`);
|
||||
}
|
||||
return delay;
|
||||
};
|
||||
}
|
||||
|
|
@ -42,6 +42,210 @@ function deliverSequencedMessage(
|
|||
}
|
||||
|
||||
describe('RedisEventTransport', () => {
|
||||
it.each([
|
||||
{ mode: 'ordinary', removal: 'unsubscribe' },
|
||||
{ mode: 'frontier', removal: 'unsubscribe' },
|
||||
{ mode: 'replay', removal: 'unsubscribe' },
|
||||
{ mode: 'ordinary', removal: 'cleanup' },
|
||||
{ mode: 'frontier', removal: 'cleanup' },
|
||||
{ mode: 'replay', removal: 'cleanup' },
|
||||
] as const)(
|
||||
'rejects $mode readiness first awaited after $removal and destroy',
|
||||
async ({ mode, removal }) => {
|
||||
const publisher = createMockPublisher();
|
||||
const subscriber = createMockSubscriber();
|
||||
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
|
||||
const transport = new RedisEventTransport(
|
||||
publisher as unknown as Redis,
|
||||
subscriber as unknown as Redis,
|
||||
);
|
||||
const options =
|
||||
mode === 'replay'
|
||||
? { replay: { items: 10, bytes: 4096, ttlMs: 1000 } }
|
||||
: { deferSequenceDelivery: true, captureSequenceFrontier: mode === 'frontier' };
|
||||
const attachment = transport.subscribe(
|
||||
'closed-before-await',
|
||||
{ onChunk: jest.fn() },
|
||||
options,
|
||||
);
|
||||
|
||||
if (removal === 'unsubscribe') attachment.unsubscribe();
|
||||
else transport.cleanup('closed-before-await');
|
||||
transport.destroy();
|
||||
|
||||
await expect(attachment.ready).rejects.toThrow('Redis event transport is closed.');
|
||||
expect(publisher.eval).not.toHaveBeenCalled();
|
||||
expect(transport.getTrackedStreamIds()).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['abort', 'preempt'] as const)(
|
||||
'rejects a pending %s registration explicitly awaited after destroy',
|
||||
async (kind) => {
|
||||
const subscriber = createMockSubscriber();
|
||||
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
|
||||
const transport = new RedisEventTransport(
|
||||
createMockPublisher() as unknown as Redis,
|
||||
subscriber as unknown as Redis,
|
||||
);
|
||||
const registration =
|
||||
kind === 'abort'
|
||||
? transport.onAbort('closed-control', jest.fn())
|
||||
: transport.onPreempt('closed-control', jest.fn());
|
||||
|
||||
transport.cleanup('closed-control');
|
||||
transport.destroy();
|
||||
|
||||
await expect(registration).rejects.toThrow('Redis event transport is closed.');
|
||||
expect(transport.getTrackedStreamIds()).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it('rejects both frontier readiness and synchronization when removed before their first await', async () => {
|
||||
const publisher = createMockPublisher();
|
||||
const subscriber = createMockSubscriber();
|
||||
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
|
||||
const transport = new RedisEventTransport(
|
||||
publisher as unknown as Redis,
|
||||
subscriber as unknown as Redis,
|
||||
);
|
||||
const attachment = transport.subscribe(
|
||||
'closed-frontier-sync',
|
||||
{ onChunk: jest.fn() },
|
||||
{
|
||||
deferSequenceDelivery: true,
|
||||
captureSequenceFrontier: true,
|
||||
},
|
||||
);
|
||||
const synchronized = attachment.syncReorderBuffer?.();
|
||||
|
||||
attachment.unsubscribe();
|
||||
transport.destroy();
|
||||
|
||||
await Promise.all([
|
||||
expect(attachment.ready).rejects.toThrow('Redis event transport is closed.'),
|
||||
expect(synchronized).rejects.toThrow('Redis event transport is closed.'),
|
||||
]);
|
||||
expect(publisher.eval).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['frontier', 'replay'] as const)(
|
||||
'preserves the original %s datastore readiness error',
|
||||
async (mode) => {
|
||||
const publisher = createMockPublisher();
|
||||
const failure = new Error('fixture EVAL failure');
|
||||
publisher.eval.mockRejectedValueOnce(failure);
|
||||
const transport = new RedisEventTransport(
|
||||
publisher as unknown as Redis,
|
||||
createMockSubscriber() as unknown as Redis,
|
||||
);
|
||||
const options =
|
||||
mode === 'replay'
|
||||
? { replay: { items: 10, bytes: 4096, ttlMs: 1000 } }
|
||||
: { deferSequenceDelivery: true, captureSequenceFrontier: true };
|
||||
const attachment = transport.subscribe('failed-readiness', { onChunk: jest.fn() }, options);
|
||||
try {
|
||||
await expect(attachment.ready).rejects.toBe(failure);
|
||||
} finally {
|
||||
attachment.unsubscribe();
|
||||
transport.destroy();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('preserves a GET failure even when cleanup removes the stream before synchronization settles', async () => {
|
||||
const publisher = createMockPublisher();
|
||||
let failRead!: (error: Error) => void;
|
||||
publisher.get.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((_resolve, reject) => {
|
||||
failRead = reject;
|
||||
}),
|
||||
);
|
||||
const transport = new RedisEventTransport(
|
||||
publisher as unknown as Redis,
|
||||
createMockSubscriber() as unknown as Redis,
|
||||
);
|
||||
const attachment = transport.subscribe('removed-sync', { onChunk: jest.fn() });
|
||||
await attachment.ready;
|
||||
const synchronized = attachment.syncReorderBuffer?.();
|
||||
transport.cleanup('removed-sync');
|
||||
const failure = new Error('fixture GET failure');
|
||||
|
||||
failRead(failure);
|
||||
|
||||
await expect(synchronized).rejects.toBe(failure);
|
||||
transport.destroy();
|
||||
});
|
||||
|
||||
it('cancels a pending attachment locally and disconnects its subscriber only once', async () => {
|
||||
jest.useFakeTimers();
|
||||
const publisher = createMockPublisher();
|
||||
const subscriber = { ...createMockSubscriber(), disconnect: jest.fn() };
|
||||
subscriber.subscribe.mockImplementation(() => new Promise(() => undefined));
|
||||
const transport = new RedisEventTransport(
|
||||
publisher as unknown as Redis,
|
||||
subscriber as unknown as Redis,
|
||||
);
|
||||
try {
|
||||
const attachment = transport.subscribe('unacknowledged-close', { onChunk: jest.fn() });
|
||||
const ready = attachment.ready?.then(
|
||||
() => 'ready',
|
||||
(error: unknown) => error,
|
||||
);
|
||||
|
||||
transport.destroy();
|
||||
transport.destroy();
|
||||
|
||||
expect(await ready).toBeInstanceOf(Error);
|
||||
expect(subscriber.disconnect).toHaveBeenCalledTimes(1);
|
||||
expect(subscriber.unsubscribe).not.toHaveBeenCalled();
|
||||
jest.runAllTicks();
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
expect(() => transport.subscribe('after-close', { onChunk: jest.fn() })).toThrow(
|
||||
'Redis event transport is closed.',
|
||||
);
|
||||
} finally {
|
||||
transport.destroy();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not publish a frontier or reopen a channel after destruction during subscribe', async () => {
|
||||
const publisher = createMockPublisher();
|
||||
const subscriber = createMockSubscriber();
|
||||
let acknowledge!: () => void;
|
||||
subscriber.subscribe.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
acknowledge = resolve;
|
||||
}),
|
||||
);
|
||||
const transport = new RedisEventTransport(
|
||||
publisher as unknown as Redis,
|
||||
subscriber as unknown as Redis,
|
||||
);
|
||||
const attachment = transport.subscribe(
|
||||
'destroyed-attachment',
|
||||
{ onChunk: jest.fn() },
|
||||
{
|
||||
deferSequenceDelivery: true,
|
||||
captureSequenceFrontier: true,
|
||||
},
|
||||
);
|
||||
const ready = attachment.ready?.then(
|
||||
() => 'ready',
|
||||
(error: unknown) => error,
|
||||
);
|
||||
|
||||
transport.destroy();
|
||||
acknowledge();
|
||||
|
||||
expect(await ready).toBeInstanceOf(Error);
|
||||
expect(publisher.eval).not.toHaveBeenCalled();
|
||||
expect(transport.getTrackedStreamIds()).toEqual([]);
|
||||
});
|
||||
|
||||
it('delivers tagged events and preserves legacy in-memory callback arity', () => {
|
||||
const transport = new InMemoryEventTransport();
|
||||
const onChunk = jest.fn();
|
||||
|
|
@ -2234,7 +2438,7 @@ describe('RedisEventTransport', () => {
|
|||
);
|
||||
|
||||
const streamId = 'reorder-abort-listener-reuse-test';
|
||||
transport.onAbort(streamId, () => {});
|
||||
const disposeAbort = await transport.onAbort(streamId, () => {});
|
||||
|
||||
const messageHandler = getMessageHandler(mockSubscriber);
|
||||
const channel = `stream:{${streamId}}:events`;
|
||||
|
|
@ -2255,6 +2459,7 @@ describe('RedisEventTransport', () => {
|
|||
|
||||
expect(secondRunChunks.map((chunk) => (chunk as { index: number }).index)).toEqual([0]);
|
||||
|
||||
disposeAbort();
|
||||
transport.destroy();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1127,7 +1127,7 @@ describe('RedisEventTransport Integration Tests', () => {
|
|||
let abortReceived = false;
|
||||
|
||||
// Register abort callback
|
||||
transport.onAbort(streamId, () => {
|
||||
const disposeAbort = await transport.onAbort(streamId, () => {
|
||||
abortReceived = true;
|
||||
});
|
||||
|
||||
|
|
@ -1142,6 +1142,7 @@ describe('RedisEventTransport Integration Tests', () => {
|
|||
|
||||
expect(abortReceived).toBe(true);
|
||||
|
||||
disposeAbort();
|
||||
transport.destroy();
|
||||
subscriber.disconnect();
|
||||
});
|
||||
|
|
@ -1307,10 +1308,10 @@ describe('RedisEventTransport Integration Tests', () => {
|
|||
let callback2Called = false;
|
||||
|
||||
// Multiple abort callbacks
|
||||
transport.onAbort(streamId, () => {
|
||||
const disposeFirst = await transport.onAbort(streamId, () => {
|
||||
callback1Called = true;
|
||||
});
|
||||
transport.onAbort(streamId, () => {
|
||||
const disposeSecond = await transport.onAbort(streamId, () => {
|
||||
callback2Called = true;
|
||||
});
|
||||
|
||||
|
|
@ -1323,6 +1324,8 @@ describe('RedisEventTransport Integration Tests', () => {
|
|||
expect(callback1Called).toBe(true);
|
||||
expect(callback2Called).toBe(true);
|
||||
|
||||
disposeFirst();
|
||||
disposeSecond();
|
||||
transport.destroy();
|
||||
subscriber.disconnect();
|
||||
});
|
||||
|
|
@ -1341,7 +1344,7 @@ describe('RedisEventTransport Integration Tests', () => {
|
|||
const streamId = `abort-cleanup-${Date.now()}`;
|
||||
let abortReceived = false;
|
||||
|
||||
transport.onAbort(streamId, () => {
|
||||
const disposeAbort = await transport.onAbort(streamId, () => {
|
||||
abortReceived = true;
|
||||
});
|
||||
|
||||
|
|
@ -1349,6 +1352,7 @@ describe('RedisEventTransport Integration Tests', () => {
|
|||
|
||||
// Cleanup the stream
|
||||
transport.cleanup(streamId);
|
||||
disposeAbort();
|
||||
|
||||
// Emit abort after cleanup
|
||||
transport.emitAbort(streamId);
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => {
|
|||
transport.destroy();
|
||||
});
|
||||
|
||||
test('abort callback survives across reconnect cycles', () => {
|
||||
test('abort callback survives across reconnect cycles', async () => {
|
||||
const mockPublisher = createMockPublisher();
|
||||
const mockSubscriber = {
|
||||
on: jest.fn(),
|
||||
|
|
@ -94,7 +94,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => {
|
|||
let abortCallbackFired = false;
|
||||
|
||||
// Register abort callback (simulates what createJob does)
|
||||
transport.onAbort(streamId, () => {
|
||||
const disposeAbort = await transport.onAbort(streamId, () => {
|
||||
abortCallbackFired = true;
|
||||
});
|
||||
|
||||
|
|
@ -115,6 +115,7 @@ describe('Reconnect Reorder Buffer Desync (Regression)', () => {
|
|||
// Abort callback should fire — it was preserved across the reconnect
|
||||
expect(abortCallbackFired).toBe(true);
|
||||
|
||||
disposeAbort();
|
||||
sub2.unsubscribe();
|
||||
transport.destroy();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -251,6 +251,7 @@ interface SubscriptionFrontierWaiter {
|
|||
|
||||
interface ChannelSubscriptionState {
|
||||
ready: Promise<void>;
|
||||
cancel?: () => void;
|
||||
phase: 'pending' | 'active';
|
||||
/** A timed-out predecessor became active while this replacement was pending. */
|
||||
fallbackActive: boolean;
|
||||
|
|
@ -304,6 +305,12 @@ interface StreamSubscribers {
|
|||
* ```
|
||||
*/
|
||||
export class RedisEventTransport implements IEventTransport {
|
||||
private destroyed = false;
|
||||
|
||||
private assertOpen(): void {
|
||||
if (this.destroyed) throw new Error('Redis event transport is closed.');
|
||||
}
|
||||
|
||||
/** Redis client for publishing events */
|
||||
private publisher: Redis | Cluster;
|
||||
/** Redis client for subscribing to events (separate connection required) */
|
||||
|
|
@ -312,6 +319,8 @@ export class RedisEventTransport implements IEventTransport {
|
|||
private streams = new Map<string, StreamSubscribers>();
|
||||
/** Track channel subscription state: resolved promise = active, pending = in-flight */
|
||||
private channelSubscriptions = new Map<string, ChannelSubscriptionState>();
|
||||
/** Pending waits outlive channel removal until they settle or the transport closes. */
|
||||
private pendingSubscriptions = new Set<ChannelSubscriptionState>();
|
||||
/** Counter for generating unique subscriber IDs */
|
||||
private subscriberIdCounter = 0;
|
||||
/** Coalescable chunk publications awaiting their window flush, per stream */
|
||||
|
|
@ -338,6 +347,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
}
|
||||
|
||||
private getOrCreateStreamState(streamId: string): StreamSubscribers {
|
||||
this.assertOpen();
|
||||
const existing = this.streams.get(streamId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
|
|
@ -353,6 +363,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
streamId: string,
|
||||
replay = false,
|
||||
): Promise<number | ReplaySnapshot> {
|
||||
this.assertOpen();
|
||||
const subscriptionFrontierId = randomUUID();
|
||||
let operationTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const observed = new Promise<void>((resolve, reject) => {
|
||||
|
|
@ -396,6 +407,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
operationTimeout.unref?.();
|
||||
});
|
||||
const [raw] = await Promise.race([operation, operationDeadline]);
|
||||
this.assertOpen();
|
||||
const counter = replay ? (raw as [string, string[]])[0] : raw;
|
||||
const parsed = counter != null ? parseInt(String(counter), 10) : 0;
|
||||
const frontier = Number.isNaN(parsed) ? 0 : parsed;
|
||||
|
|
@ -441,7 +453,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
|
||||
// Set up message handler for all subscriptions
|
||||
this.subscriber.on('message', (channel: string, message: string) => {
|
||||
this.handleMessage(channel, message);
|
||||
if (!this.destroyed) this.handleMessage(channel, message);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -677,6 +689,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
}
|
||||
|
||||
private ensureChannelSubscription(streamId: string): Promise<void> {
|
||||
this.assertOpen();
|
||||
const channel = CHANNELS.events(streamId);
|
||||
const existing = this.channelSubscriptions.get(channel);
|
||||
if (existing) {
|
||||
|
|
@ -692,6 +705,10 @@ export class RedisEventTransport implements IEventTransport {
|
|||
fallbackActive: false,
|
||||
};
|
||||
const deadline = new Promise<never>((_, reject) => {
|
||||
state.cancel = () => {
|
||||
clearTimeout(timeout);
|
||||
reject(new Error('Redis event transport is closed.'));
|
||||
};
|
||||
timeout = setTimeout(() => {
|
||||
expired = true;
|
||||
reject(new Error(`Timed out synchronizing Redis subscription for ${channel}`));
|
||||
|
|
@ -699,6 +716,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
timeout.unref?.();
|
||||
});
|
||||
const ready = Promise.race([operation, deadline]).then(() => {
|
||||
this.assertOpen();
|
||||
state.phase = 'active';
|
||||
state.fallbackActive = false;
|
||||
logger.debug(`[RedisEventTransport] Subscription active for channel ${channel}`);
|
||||
|
|
@ -707,7 +725,7 @@ export class RedisEventTransport implements IEventTransport {
|
|||
operation.then(
|
||||
() => {
|
||||
if (timeout != null) clearTimeout(timeout);
|
||||
if (expired) {
|
||||
if (expired && !this.destroyed) {
|
||||
const current = this.channelSubscriptions.get(channel);
|
||||
if (current == null || current === state) {
|
||||
if (current === state) this.channelSubscriptions.delete(channel);
|
||||
|
|
@ -724,13 +742,19 @@ export class RedisEventTransport implements IEventTransport {
|
|||
},
|
||||
);
|
||||
this.channelSubscriptions.set(channel, state);
|
||||
void ready.catch((err) => {
|
||||
if (this.channelSubscriptions.get(channel) === state) {
|
||||
this.channelSubscriptions.delete(channel);
|
||||
if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel);
|
||||
}
|
||||
logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err);
|
||||
});
|
||||
this.pendingSubscriptions.add(state);
|
||||
void ready.then(
|
||||
() => this.pendingSubscriptions.delete(state),
|
||||
(err) => {
|
||||
this.pendingSubscriptions.delete(state);
|
||||
if (this.destroyed) return;
|
||||
if (this.channelSubscriptions.get(channel) === state) {
|
||||
this.channelSubscriptions.delete(channel);
|
||||
if (state.fallbackActive) this.trackOrReleaseActiveChannel(streamId, channel);
|
||||
}
|
||||
logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err);
|
||||
},
|
||||
);
|
||||
return ready;
|
||||
}
|
||||
|
||||
|
|
@ -1943,11 +1967,15 @@ export class RedisEventTransport implements IEventTransport {
|
|||
* Destroy all resources.
|
||||
*/
|
||||
destroy(): void {
|
||||
for (const streamId of new Set(
|
||||
[...this.subscriptionFrontierWaiters.values()].map((waiter) => waiter.streamId),
|
||||
)) {
|
||||
this.releaseSubscriptionFrontiers(streamId);
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const waiter of this.subscriptionFrontierWaiters.values()) {
|
||||
clearTimeout(waiter.timeout);
|
||||
waiter.reject(new Error('Redis event transport is closed.'));
|
||||
}
|
||||
this.subscriptionFrontierWaiters.clear();
|
||||
for (const state of this.pendingSubscriptions) state.cancel?.();
|
||||
this.pendingSubscriptions.clear();
|
||||
for (const streamId of this.pendingBatches.keys()) {
|
||||
this.discardCoalescedChunks(streamId);
|
||||
}
|
||||
|
|
@ -1969,10 +1997,6 @@ export class RedisEventTransport implements IEventTransport {
|
|||
state.abortAckWaiters.clear();
|
||||
}
|
||||
|
||||
for (const channel of this.channelSubscriptions.keys()) {
|
||||
this.subscriber.unsubscribe(channel).catch(() => {});
|
||||
}
|
||||
|
||||
this.channelSubscriptions.clear();
|
||||
this.streams.clear();
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue