LibreChat/api/server/services/Endpoints/agents/subagentThreadStore.spec.js
Danny Avila 2d794a0439
💓 fix: Detect Silently Dead Redis Sockets With Deadline-Bounded Heartbeats (#15539)
* fix: Detect silently dead Redis sockets with deadline-bounded heartbeats

A Redis peer can vanish without a FIN or RST (a dropped NAT entry, a proxy
failover, a migrated VM). The socket stays `ready` from ioredis's point of
view, no error fires, and every command on it waits for the kernel's
retransmission timeout — about fifteen minutes at Linux defaults.

Observed in production: while the main client's socket was dead,
POST /api/agents hung until Cloudflare returned a 524; while the subscriber's
socket was dead, every attach failed with "Timed out synchronizing Redis
subscription" for sixteen minutes across five conversations.

`REDIS_PING_INTERVAL` could not help — it awaits ping() with no deadline, so
the probe just joins everything else that is waiting.

- Add startRedisHeartbeat: races a PING against a deadline and destroys the
  socket when the probe goes unanswered, so ioredis reconnects, replays queued
  commands, and re-subscribes a subscriber's channels.
- Destroy the stream rather than disconnect(true), whose FIN a vanished peer
  never acknowledges; cluster clients fall back to the regular reconnect.
- Add createIoRedisSubscriber so every dedicated subscriber gets an error
  listener and its own heartbeat, and route the generation-stream and both
  subagent-routing subscribers through it.
- Set ioredis keepAlive from REDIS_KEEP_ALIVE (default 10s).
- Add REDIS_PING_TIMEOUT, REDIS_SUBSCRIBER_PING_INTERVAL and REDIS_KEEP_ALIVE,
  documented in .env.example.
- Give the fake RESP server a silent mode and a SUBSCRIBE reply so the
  dead-peer case can be tested against real clients.

* fix: Probe each cluster node and refuse a non-positive heartbeat deadline

A cluster-level PING is routed to one node, so a healthy reply proved nothing
about the other sockets. The heartbeat now resolves its targets each tick —
every node for a cluster, the client itself otherwise — and races an
independent deadline per target, tearing down only the socket that stops
answering. A non-positive deadline disables the heartbeat with a warning
instead of forcing a reconnect on every tick, and the single-probe test no
longer depends on wall-clock slack.
2026-09-03 09:45:16 -04:00

132 lines
4.9 KiB
JavaScript

const mockTaskStore = {
configureTaskControlTransport: jest.fn().mockResolvedValue(undefined),
configureActivityStream: jest.fn(),
prepareActivityForShutdown: jest.fn(),
destroyTaskControlTransport: jest.fn().mockResolvedValue(undefined),
destroyActivityStream: jest.fn(),
};
const mockCompletionWakeupHandler = jest.fn().mockResolvedValue(undefined);
jest.mock('@librechat/api', () => ({
cacheConfig: { USE_REDIS: true, REDIS_KEY_PREFIX: 'test:' },
ioredisClient: { duplicate: jest.fn() },
registerShutdownTask: jest.fn(),
duplicateIoRedisClient: jest.fn(),
createIoRedisSubscriber: jest.fn(),
createSubagentThreadTaskStore: jest.fn(() => mockTaskStore),
createSubagentCompletionWakeupHandler: jest.fn(() => mockCompletionWakeupHandler),
RedisSubagentTaskControlTransport: jest.fn(),
RedisEventTransport: jest.fn(),
SubagentActivityStream: jest.fn(),
}));
jest.mock('~/models', () => ({
acquireSubagentThreadLease: jest.fn(),
claimSubagentTaskResult: jest.fn(),
releaseSubagentTaskResultClaim: jest.fn(),
countActiveSubagentThreadLeases: jest.fn(),
deleteConvos: jest.fn(),
deleteMessages: jest.fn(),
getConvo: jest.fn(),
getSubagentTaskControlReplay: jest.fn(),
getMessages: jest.fn(),
listActiveSubagentThreadLeases: jest.fn(),
recordSubagentTaskControlReceipt: jest.fn(),
releaseSubagentThreadLease: jest.fn(),
reserveSubagentThread: jest.fn(),
renewSubagentThreadLease: jest.fn(),
saveConvo: jest.fn(),
saveMessage: jest.fn(),
isSubagentOwnerAdmissible: jest.fn(),
fenceSubagentAdmission: jest.fn(),
renewSubagentAdmission: jest.fn(),
releaseSubagentAdmission: jest.fn(),
}));
jest.mock('../../Agents/triggers', () => ({
enqueueAgentTrigger: jest.fn(),
}));
const {
ioredisClient,
registerShutdownTask,
duplicateIoRedisClient,
createIoRedisSubscriber,
createSubagentThreadTaskStore,
} = require('@librechat/api');
const subagentThreadTaskStore = require('./subagentThreadStore');
const { configureSubagentTaskRouting } = subagentThreadTaskStore;
const taskStoreOptions = createSubagentThreadTaskStore.mock.calls[0][1];
const taskStoreMethods = createSubagentThreadTaskStore.mock.calls[0][0];
const db = require('~/models');
const activityPrepareRegistration = registerShutdownTask.mock.calls.find(
([name]) => name === 'subagent activity streams prepare',
);
const taskStoreShutdownRegistration = registerShutdownTask.mock.calls.find(
([name]) => name === 'subagent task store',
);
describe('subagent thread Redis lifecycle', () => {
it('wires durable control receipt persistence into the host store', () => {
expect(taskStoreMethods.recordSubagentTaskControlReceipt).toBe(
db.recordSubagentTaskControlReceipt,
);
expect(taskStoreMethods.getSubagentTaskControlReplay).toBe(db.getSubagentTaskControlReplay);
});
it('pre-registers completion wakeups for every prepared task', async () => {
await taskStoreOptions.onTaskPrepared({ taskId: 'task-1' });
expect(mockCompletionWakeupHandler).toHaveBeenCalledWith({ taskId: 'task-1' });
});
it('registers local task-store quiescence independently of optional Redis setup', () => {
expect(taskStoreShutdownRegistration).toEqual([
'subagent task store',
expect.any(Function),
{ priority: 90 },
]);
});
it('closes activity SSE before drain and disconnects its subscriber after drain', async () => {
const taskSubscriber = { disconnect: jest.fn() };
const activitySubscriber = { disconnect: jest.fn() };
const taskPublisher = { disconnect: jest.fn() };
const activityPublisher = { disconnect: jest.fn() };
createIoRedisSubscriber
.mockReturnValueOnce(taskSubscriber)
.mockReturnValueOnce(activitySubscriber);
duplicateIoRedisClient
.mockReturnValueOnce(taskPublisher)
.mockReturnValueOnce(activityPublisher);
await configureSubagentTaskRouting();
expect(createIoRedisSubscriber.mock.calls).toEqual([
[ioredisClient, '[SubagentTaskRouting] task subscriber'],
[ioredisClient, '[SubagentTaskRouting] activity subscriber'],
]);
expect(activityPrepareRegistration).toEqual([
'subagent activity streams prepare',
expect.any(Function),
{ phase: 'pre-drain', priority: 100 },
]);
expect(taskStoreShutdownRegistration).toEqual([
'subagent task store',
expect.any(Function),
{ priority: 90 },
]);
const prepare = activityPrepareRegistration[1];
prepare();
expect(mockTaskStore.prepareActivityForShutdown).toHaveBeenCalledTimes(1);
const shutdown = taskStoreShutdownRegistration[1];
await shutdown();
expect(mockTaskStore.destroyTaskControlTransport).toHaveBeenCalledTimes(1);
expect(mockTaskStore.destroyActivityStream).toHaveBeenCalledTimes(1);
expect(taskPublisher.disconnect).toHaveBeenCalledTimes(1);
expect(activitySubscriber.disconnect).toHaveBeenCalledTimes(1);
expect(activityPublisher.disconnect).toHaveBeenCalledTimes(1);
});
});