LibreChat/packages/api/src/cache/redisClients.ts
Danny Avila 2d794a0439
💓 fix: Detect Silently Dead Redis Sockets With Deadline-Bounded Heartbeats (#15539)
* fix: Detect silently dead Redis sockets with deadline-bounded heartbeats

A Redis peer can vanish without a FIN or RST (a dropped NAT entry, a proxy
failover, a migrated VM). The socket stays `ready` from ioredis's point of
view, no error fires, and every command on it waits for the kernel's
retransmission timeout — about fifteen minutes at Linux defaults.

Observed in production: while the main client's socket was dead,
POST /api/agents hung until Cloudflare returned a 524; while the subscriber's
socket was dead, every attach failed with "Timed out synchronizing Redis
subscription" for sixteen minutes across five conversations.

`REDIS_PING_INTERVAL` could not help — it awaits ping() with no deadline, so
the probe just joins everything else that is waiting.

- Add startRedisHeartbeat: races a PING against a deadline and destroys the
  socket when the probe goes unanswered, so ioredis reconnects, replays queued
  commands, and re-subscribes a subscriber's channels.
- Destroy the stream rather than disconnect(true), whose FIN a vanished peer
  never acknowledges; cluster clients fall back to the regular reconnect.
- Add createIoRedisSubscriber so every dedicated subscriber gets an error
  listener and its own heartbeat, and route the generation-stream and both
  subagent-routing subscribers through it.
- Set ioredis keepAlive from REDIS_KEEP_ALIVE (default 10s).
- Add REDIS_PING_TIMEOUT, REDIS_SUBSCRIBER_PING_INTERVAL and REDIS_KEEP_ALIVE,
  documented in .env.example.
- Give the fake RESP server a silent mode and a SUBSCRIBE reply so the
  dead-peer case can be tested against real clients.

* fix: Probe each cluster node and refuse a non-positive heartbeat deadline

A cluster-level PING is routed to one node, so a healthy reply proved nothing
about the other sockets. The heartbeat now resolves its targets each tick —
every node for a cluster, the client itself otherwise — and races an
independent deadline per target, tearing down only the socket that stops
answering. A non-positive deadline disables the heartbeat with a warning
instead of forcing a reconnect on every tick, and the single-probe test no
longer depends on wall-clock slack.
2026-09-03 09:45:16 -04:00

301 lines
11 KiB
TypeScript

import IoRedis from 'ioredis';
import calculateSlot from 'cluster-key-slot';
import { logger } from '@librechat/data-schemas';
import { createClient, createCluster } from '@keyv/redis';
import type { ScanOptions } from '@redis/client/dist/lib/commands/SCAN';
import type { RedisClientType, RedisClusterType } from '@redis/client';
import type { Redis, Cluster } from 'ioredis';
import type { ReadonlyRecoveryHandler } from './recovery';
import { createReadonlyRecovery, isReadonlyReplicaError } from './recovery';
import { startRedisHeartbeat } from './heartbeat';
import { cacheConfig } from './cacheConfig';
const urls = cacheConfig.REDIS_URI?.split(',').map((uri) => new URL(uri)) || [];
const username = urls?.[0]?.username || cacheConfig.REDIS_USERNAME;
const password = urls?.[0]?.password || cacheConfig.REDIS_PASSWORD;
const ca = cacheConfig.REDIS_CA;
const protocols = new Set(urls.map((url) => url.protocol));
const useTls = urls[0]?.protocol === 'rediss:';
const isRedisCluster = urls.length !== 1 || cacheConfig.USE_REDIS_CLUSTER;
if (cacheConfig.USE_REDIS && protocols.size > 1) {
throw new Error('All REDIS_URI entries must use the same protocol');
}
if (cacheConfig.USE_REDIS && ca && !useTls) {
throw new Error('REDIS_CA requires REDIS_URI to use rediss://');
}
let resolveKeyvRedisClientReady: (() => void) | undefined;
let rejectKeyvRedisClientReady: ((reason?: unknown) => void) | undefined;
const keyvRedisClientReady: Promise<void> | null = cacheConfig.USE_REDIS
? new Promise<void>((resolve, reject) => {
resolveKeyvRedisClientReady = resolve;
rejectKeyvRedisClientReady = reject;
})
: null;
/** Waits for the stable shared Keyv Redis readiness gate. */
async function waitForKeyvRedisClient(): Promise<void> {
await keyvRedisClientReady;
}
let ioredisClient: Redis | Cluster | null = null;
if (cacheConfig.USE_REDIS) {
const redisOptions: Record<string, unknown> = {
username: username,
password: password,
tls: useTls ? { ca: ca ?? undefined } : undefined,
keyPrefix: `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR}`,
maxListeners: cacheConfig.REDIS_MAX_LISTENERS,
retryStrategy: (times: number) => {
if (
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
) {
logger.error(
`ioredis giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
);
return null;
}
const base = Math.min(Math.pow(2, times) * 50, cacheConfig.REDIS_RETRY_MAX_DELAY);
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
logger.info(`ioredis reconnecting... attempt ${times}, delay ${delay}ms`);
return delay;
},
reconnectOnError: (err: Error) => {
if (isReadonlyReplicaError(err)) {
logger.warn('ioredis reconnecting due to READONLY error');
return 2; // Return retry delay instead of boolean
}
return false;
},
enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
maxRetriesPerRequest: 3,
keepAlive: cacheConfig.REDIS_KEEP_ALIVE,
};
ioredisClient = !isRedisCluster
? new IoRedis(cacheConfig.REDIS_URI!, redisOptions)
: new IoRedis.Cluster(
urls.map((url) => ({ host: url.hostname, port: parseInt(url.port, 10) || 6379 })),
{
...(cacheConfig.REDIS_USE_ALTERNATIVE_DNS_LOOKUP
? {
dnsLookup: (
address: string,
callback: (err: Error | null, address: string) => void,
) => callback(null, address),
}
: {}),
redisOptions,
clusterRetryStrategy: (times: number) => {
if (
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
) {
logger.error(
`ioredis cluster giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
);
return null;
}
const base = Math.min(Math.pow(2, times) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
logger.info(`ioredis cluster reconnecting... attempt ${times}, delay ${delay}ms`);
return delay;
},
enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
},
);
ioredisClient.on('error', (err) => {
logger.error('ioredis client error:', err);
});
ioredisClient.on('connect', () => {
logger.info('ioredis client connected');
});
ioredisClient.on('ready', () => {
logger.info('ioredis client ready');
});
ioredisClient.on('reconnecting', (delay: number) => {
logger.info(`ioredis client reconnecting in ${delay}ms`);
});
ioredisClient.on('close', () => {
logger.warn('ioredis client connection closed');
});
/** Deadline-bounded keepalive PINGs; an unanswered probe forces a reconnect. */
if (cacheConfig.REDIS_PING_INTERVAL > 0) {
startRedisHeartbeat({
client: ioredisClient,
intervalMs: cacheConfig.REDIS_PING_INTERVAL * 1000,
timeoutMs: cacheConfig.REDIS_PING_TIMEOUT,
label: 'ioredis client',
});
}
}
let keyvRedisClient: RedisClientType | RedisClusterType | null = null;
let recoverKeyvRedisClient: ReadonlyRecoveryHandler | undefined;
/**
* Routes a Keyv Redis client error through READONLY failover recovery.
* Cluster clients follow MOVED to the owning master on their own, so the hook
* only exists for the standalone client. Returns whether a reconnect was started.
*/
function handleKeyvRedisError(error: unknown): boolean {
return recoverKeyvRedisClient?.(error) ?? false;
}
type RedisEvalOptions = { keys: string[]; arguments: string[] };
async function runKeyvRedisScript(
client: RedisClientType | RedisClusterType,
script: string,
options: RedisEvalOptions,
): Promise<unknown> {
if (!('masters' in client) || options.keys.length === 0) {
return client.eval(script, options);
}
const slot = calculateSlot(options.keys[0]);
if (options.keys.some((key) => calculateSlot(key) !== slot)) {
throw new Error('Redis catalog script keys must share one cluster slot');
}
const master = client.getSlotMaster(slot);
const nodeClient = await client.nodeClient(master);
return nodeClient.eval(script, options);
}
/**
* Runs a Lua script on the master that owns its keys. Node Redis can execute a
* cluster EVAL through an arbitrary node while the slot map is settling, which
* leaks a MOVED reply instead of following it. Catalog scripts are deliberately
* single-slot, so selecting the owning master also makes that invariant explicit.
*
* Script failures reject straight to the caller without passing through any
* client error event, so READONLY replies are routed to failover recovery here.
*/
async function evalKeyvRedisScript(script: string, options: RedisEvalOptions): Promise<unknown> {
await waitForKeyvRedisClient();
if (!keyvRedisClient) {
throw new Error('Keyv Redis client is not configured');
}
try {
return await runKeyvRedisScript(keyvRedisClient, script, options);
} catch (error) {
handleKeyvRedisError(error);
throw error;
}
}
if (cacheConfig.USE_REDIS) {
/**
* ** WARNING ** Keyv Redis client does not support Prefix like ioredis above.
* The prefix feature will be handled by the Keyv-Redis store in cacheFactory.js
*/
const redisOptions: Record<string, unknown> = {
username,
password,
socket: {
...(isRedisCluster ? { tls: useTls } : {}),
...(ca ? { ca } : {}),
connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
reconnectStrategy: (retries: number) => {
if (
cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
retries > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
) {
logger.error(
`@keyv/redis client giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
);
return new Error('Max reconnection attempts reached');
}
const base = Math.min(Math.pow(2, retries) * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
const jitter = Math.floor(Math.random() * Math.min(base, 1000));
const delay = Math.min(base + jitter, cacheConfig.REDIS_RETRY_MAX_DELAY);
logger.info(`@keyv/redis reconnecting... attempt ${retries}, delay ${delay}ms`);
return delay;
},
},
disableOfflineQueue: !cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
...(cacheConfig.REDIS_PING_INTERVAL > 0
? { pingInterval: cacheConfig.REDIS_PING_INTERVAL * 1000 }
: {}),
};
keyvRedisClient = !isRedisCluster
? createClient({ url: cacheConfig.REDIS_URI, ...redisOptions })
: createCluster({
rootNodes: urls.map((url) => ({ url: url.href })),
defaults: redisOptions,
});
// Add scanIterator method to cluster client for API consistency with standalone client
if (!('scanIterator' in keyvRedisClient)) {
const clusterClient = keyvRedisClient as RedisClusterType;
(keyvRedisClient as unknown as RedisClientType).scanIterator = async function* (
options?: ScanOptions,
) {
const masters = clusterClient.masters;
for (const master of masters) {
const nodeClient = await clusterClient.nodeClient(master);
for await (const page of nodeClient.scanIterator(options)) {
yield page;
}
}
};
}
keyvRedisClient.setMaxListeners(cacheConfig.REDIS_MAX_LISTENERS);
if (!isRedisCluster) {
recoverKeyvRedisClient = createReadonlyRecovery({
client: keyvRedisClient as RedisClientType,
minIntervalMs: cacheConfig.REDIS_READONLY_RECOVERY_INTERVAL,
});
}
keyvRedisClient.on('error', (err) => {
logger.error('@keyv/redis client error:', err);
});
keyvRedisClient.on('connect', () => {
logger.info('@keyv/redis client connected');
});
keyvRedisClient.on('ready', () => {
logger.info('@keyv/redis client ready');
});
keyvRedisClient.on('reconnecting', () => {
logger.info('@keyv/redis client reconnecting...');
});
keyvRedisClient.on('disconnect', () => {
logger.warn('@keyv/redis client disconnected');
});
// Start connection immediately and settle the gate created before client initialization.
void keyvRedisClient.connect().then(resolveKeyvRedisClientReady, rejectKeyvRedisClientReady);
void keyvRedisClientReady?.catch((err): void => {
logger.error('@keyv/redis initial connection failed:', err);
});
}
export {
ioredisClient,
keyvRedisClient,
keyvRedisClientReady,
waitForKeyvRedisClient,
evalKeyvRedisScript,
handleKeyvRedisError,
};