🔧 fix: Fence Channel Startup and Transaction Capability (#17831)

* 🔧 fix: Fence Channel Startup and Transaction Capability

* 🔧 fix: Gate Clustered Workers Until Startup Completes
This commit is contained in:
Dustin Healy 2026-11-10 21:04:10 -08:00 • committed by GitHub
parent 1f27f5848a
commit fdf75e0174
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 227 additions and 16 deletions

View file

@ -1,9 +1,11 @@
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { execFile, execFileSync } from 'node:child_process';
import { copyFileSync, mkdtempSync, rmSync } from 'node:fs';
import { createServer } from 'node:http';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
const root = fileURLToPath(new URL('../..', import.meta.url));
const fixture = mkdtempSync(join(tmpdir(), 'channel-compose-'));
@ -48,6 +50,7 @@ function configuration(tag, source = false, shared = false, gatewayPort) {
...process.env,
UID: '1000',
GID: '1000',
PORT: '3080',
LIBRECHAT_IMAGE_TAG: tag,
SLACK_APP_ID: 'A123',
SLACK_CLIENT_ID: '123.456',
@ -121,3 +124,30 @@ assert.equal(
'3999',
);
console.log('PASS: shared Compose needs no app credentials and clears all bound-gateway settings');
// Run the shipped healthcheck against an API that listens before initialization completes.
let ready = false;
const server = createServer((req, res) => {
res.writeHead(req.url === '/health' || (req.url === '/readyz' && ready) ? 200 : 503);
res.end();
});
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
try {
const { port } = server.address();
const [kind, command, flag, script] = configuration('').api.healthcheck.test;
assert.equal(kind, 'CMD');
assert.equal(command, 'node');
assert.equal(flag, '-e');
const probe = () =>
promisify(execFile)(process.execPath, [
flag,
script.replace('127.0.0.1:3080', `127.0.0.1:${port}`),
]);
assert.equal((await fetch(`http://127.0.0.1:${port}/health`)).status, 200);
await assert.rejects(probe(), { code: 1 });
ready = true;
await probe();
console.log('PASS: gateway dependency stays unhealthy until API readiness succeeds');
} finally {
await new Promise((resolve) => server.close(resolve));
}

View file

@ -8,6 +8,10 @@ import yaml from 'js-yaml';
const readWorkflow = (file) =>
yaml.load(readFileSync(new URL(`../workflows/${file}`, import.meta.url), 'utf8'));
const manual = readWorkflow('main-image-workflow.yml');
assert.ok(
readWorkflow('retry-docker-builds.yml').on.workflow_run.workflows.includes(manual.name),
'the retry workflow must subscribe to the manual release publisher by its current name',
);
for (const workflow of [readWorkflow('tag-images.yml')]) {
const images = JSON.parse(workflow.jobs.publish.with.images);
const gateway = images.find((image) => image.image_name === 'librechat-channel-gateway');

View file

@ -17,6 +17,7 @@ on:
- '.github/scripts/test-channel-release.mjs'
- '.github/workflows/tag-images.yml'
- '.github/workflows/main-image-workflow.yml'
- '.github/workflows/retry-docker-builds.yml'
- '.github/workflows/docker-publish.yml'
- 'helm/librechat/**'
- 'docker-compose.mongo-replica.yml'
@ -40,6 +41,7 @@ on:
- '.github/scripts/test-channel-release.mjs'
- '.github/workflows/tag-images.yml'
- '.github/workflows/main-image-workflow.yml'
- '.github/workflows/retry-docker-builds.yml'
- '.github/workflows/docker-publish.yml'
- 'helm/librechat/**'
- 'docker-compose.mongo-replica.yml'

View file

@ -4,7 +4,7 @@ on:
workflow_run:
workflows:
- Docker Build Smoke Tests
- Docker Compose Build Latest Main Image Tag (Manual Dispatch)
- Publish Latest Stable Release Images (Manual Dispatch)
- Docker Dev Branch Images Build
- Docker Dev Images Build
- Docker Dev Staging Images Build

View file

@ -59,6 +59,9 @@ const {
cacheConfig,
createClusteredFileSweep,
createRequestSanitizer,
createStartupReadiness,
keyvRedisClient,
ioredisClient,
} = require('@librechat/api');
const { connectDb, indexSync } = require('~/db');
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
@ -379,6 +382,11 @@ if (cluster.isMaster) {
* Each worker runs a full Express server instance
*/
const app = express();
const readiness = createStartupReadiness({
useRedis: cacheConfig.USE_REDIS,
ioredis: ioredisClient,
keyv: keyvRedisClient,
});
app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry();
// The clustered entrypoint deliberately does not arm the v1 schedule engine,
// but an already-fired scheduled generation can still reach HITL here. Settle
@ -607,6 +615,8 @@ if (cluster.isMaster) {
/** Health check endpoint */
app.get('/health', (_req, res) => res.status(200).send('OK'));
app.get('/readyz', readiness.handler);
app.use('/api', readiness.gate);
/** Middleware */
app.use(requestContextMiddleware);
@ -764,6 +774,7 @@ if (cluster.isMaster) {
registerShutdownTask('channel feedback export', stopChannelFeedback, { priority: 100 });
registerShutdownTask('channel authority cleanup', stopChannelCleanup, { priority: 100 });
await initializeChannelCleanup();
readiness.markReady();
} catch (initErr) {
logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);
process.exit(1);

View file

@ -11,7 +11,7 @@ services:
'CMD',
'node',
'-e',
"fetch('http://127.0.0.1:${PORT:-3080}/health',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})",
"fetch('http://127.0.0.1:${PORT:-3080}/readyz',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})",
]
interval: 10s
timeout: 5s

View file

@ -1,11 +1,13 @@
import IoRedis from 'ioredis';
import express from 'express';
import request from 'supertest';
import { createClient } from '@keyv/redis';
import type { RedisClientType } from '@redis/client';
import type { Request, Response } from 'express';
import type { RespServer } from '~/cache/__tests__/resp.helper';
import type { RedisReadinessClients } from './readiness';
import { createReadinessHandler, createStartupReadiness, isRedisReady } from './readiness';
import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper';
import { createReadinessHandler, isRedisReady } from './readiness';
const clients = (overrides: Partial<RedisReadinessClients> = {}): RedisReadinessClients => ({
useRedis: true,
@ -129,3 +131,51 @@ describe('createReadinessHandler', () => {
expect(respond(handler).code).toBe(200);
});
});
describe('worker startup readiness', () => {
const worker = (redis: RedisReadinessClients = clients({ useRedis: false })) => {
const readiness = createStartupReadiness(redis);
const app = express();
app.get('/readyz', readiness.handler);
app.use('/api', readiness.gate);
app.get('/api/work', (_req, res) => {
res.sendStatus(200);
});
return { app, readiness };
};
it('refuses API work until post-listen initialization completes', async () => {
const { app, readiness } = worker();
await request(app).get('/readyz').expect(503);
await request(app)
.get('/api/work')
.expect('Retry-After', '1')
.expect(503)
.expect(({ body }) => expect(body.code).toBe('SERVER_NOT_READY'));
readiness.markReady();
await request(app).get('/readyz').expect(200);
await request(app).get('/api/work').expect(200);
});
it('keeps another starting worker closed when its sibling becomes ready', async () => {
const first = worker();
const second = worker();
first.readiness.markReady();
await request(first.app).get('/api/work').expect(200);
expect((await request(second.app).get('/readyz')).status).toBe(503);
await request(second.app).get('/api/work').expect(503);
});
it('rechecks Redis readiness for both the probe and API admission', async () => {
const keyv = { isReady: true };
const { app, readiness } = worker(clients({ keyv }));
readiness.markReady();
await request(app).get('/api/work').expect(200);
keyv.isReady = false;
expect((await request(app).get('/readyz')).status).toBe(503);
await request(app).get('/api/work').expect(503);
keyv.isReady = true;
await request(app).get('/readyz').expect(200);
await request(app).get('/api/work').expect(200);
});
});

View file

@ -1,5 +1,5 @@
import type { Request, Response, RequestHandler } from 'express';
import type { RedisClientType } from '@redis/client';
import type { Request, Response } from 'express';
import type { Redis, Cluster } from 'ioredis';
type NodeRedisReadiness = Pick<RedisClientType, 'isReady'>;
@ -63,3 +63,30 @@ export const createReadinessHandler =
}
res.status(200).send('OK');
};
/** Per-worker readiness: a healthy sibling must not admit work to an initializing worker. */
export function createStartupReadiness(redis: RedisReadinessClients): {
handler: RequestHandler;
gate: RequestHandler;
markReady: () => void;
} {
let initialized = false;
const gate: RequestHandler = (_req, res, next) => {
if (initialized && isRedisReady(redis)) {
next();
return;
}
res.set('Retry-After', '1');
res.status(503).json({
code: 'SERVER_NOT_READY',
error: 'Server is not ready. Please retry shortly.',
});
};
return {
handler: createReadinessHandler(() => initialized, redis),
gate,
markReady: () => {
initialized = true;
},
};
}

View file

@ -5,6 +5,7 @@ import {
supportsTransactions,
startRequiredTransactionSession,
TransactionUnavailableError,
withRequiredTransaction,
} from './transactions';
jest.mock('~/config/winston', () => ({ debug: jest.fn() }));
@ -140,7 +141,7 @@ describe('transaction capability against real MongoDB', () => {
it('reuses a required-session verdict instead of probing every subsequent transition', async () => {
const db = new mongoose.Mongoose();
await db.connect(repl.getUri(), { readPreference: 'secondaryPreferred' });
const starts = jest.spyOn(db, 'startSession');
const starts = jest.spyOn(db.connection.getClient(), 'startSession');
try {
const first = await startRequiredTransactionSession(db);
await first.endSession();
@ -164,7 +165,7 @@ describe('transaction capability against real MongoDB', () => {
await expect(startRequiredTransactionSession(db)).rejects.toBe(failure);
const session = await startRequiredTransactionSession(db);
await session.endSession();
expect(starts).toHaveBeenCalledTimes(3);
expect(starts).toHaveBeenCalledTimes(2); // Failed probe, then successful probe.
} finally {
await db.disconnect();
}
@ -196,4 +197,66 @@ describe('transaction capability against real MongoDB', () => {
);
expect(starts).toHaveBeenCalledTimes(1);
});
it('rejects an old pending verdict and probes the replacement client before private effects', async () => {
const db = new mongoose.Mongoose();
await db.connect(repl.getUri());
const probeSession = await db.startSession();
const commit = probeSession.commitTransaction.bind(probeSession);
let release!: () => void;
let entered!: () => void;
const paused = new Promise<void>((resolve) => (release = resolve));
const committed = new Promise<void>((resolve) => (entered = resolve));
jest.spyOn(probeSession, 'commitTransaction').mockImplementationOnce(async () => {
await commit();
entered();
await paused;
});
const starts = jest.spyOn(db, 'startSession').mockResolvedValueOnce(probeSession);
const prepare = jest.fn(async () => undefined);
const write = jest.fn(async () => 'private-effect');
try {
const stale = withRequiredTransaction(db, write, prepare);
await committed;
await db.disconnect();
await db.connect(mongo.getUri());
const current = withRequiredTransaction(db, write, prepare);
const outcomes = Promise.allSettled([stale, current]);
release();
expect(await outcomes).toEqual([
{ status: 'rejected', reason: expect.any(db.mongo.MongoNetworkError) },
{ status: 'rejected', reason: expect.any(TransactionUnavailableError) },
]);
await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf(
TransactionUnavailableError,
);
expect(starts).toHaveBeenCalledTimes(2); // The old probe and a fresh standalone probe.
expect(prepare).not.toHaveBeenCalled();
expect(write).not.toHaveBeenCalled();
} finally {
release();
await db.disconnect();
}
});
it('rechecks the client after awaiting a cached capability verdict', async () => {
const db = new mongoose.Mongoose();
await db.connect(repl.getUri());
try {
const session = await startRequiredTransactionSession(db);
await session.endSession();
const client = db.connection.getClient();
jest
.spyOn(db.connection, 'getClient')
.mockReturnValueOnce(client)
.mockReturnValueOnce(standalone.connection.getClient());
const starts = jest.spyOn(db, 'startSession');
await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf(
db.mongo.MongoNetworkError,
);
expect(starts).not.toHaveBeenCalled();
} finally {
await db.disconnect();
}
});
});

View file

@ -62,21 +62,41 @@ export const supportsTransactions = async (
}
};
const inflightProbes = new WeakMap<import('mongoose').Connection, Promise<boolean>>();
type TransactionClient = ReturnType<Connection['getClient']>;
const inflightProbes = new WeakMap<Connection, WeakMap<TransactionClient, Promise<boolean>>>();
/** Share first probes per connection. Callers cache only a successful capability verdict. */
function assertTransactionClient(
mongoose: typeof import('mongoose'),
connection: Connection,
client: TransactionClient,
): void {
if (mongoose.connection !== connection || connection.getClient() !== client) {
throw new mongoose.mongo.MongoNetworkError('MongoDB client changed during transaction setup');
}
}
/** Share first probes only for the same connection and client. */
export const getTransactionSupport = async (
mongoose: typeof import('mongoose'),
transactionSupportCache: boolean | null,
): Promise<boolean> => {
if (transactionSupportCache !== null) return transactionSupportCache;
const connection = mongoose.connection;
let probe = inflightProbes.get(connection);
if (!probe) {
probe = supportsTransactions(mongoose).finally(() => inflightProbes.delete(connection));
inflightProbes.set(connection, probe);
const client = connection.getClient();
let probes = inflightProbes.get(connection);
if (!probes) {
probes = new WeakMap();
inflightProbes.set(connection, probes);
}
return probe;
let probe = probes.get(client);
if (!probe) {
const pending = probes;
probe = supportsTransactions(mongoose).finally(() => pending.delete(client));
probes.set(client, probe);
}
const supported = await probe;
assertTransactionClient(mongoose, connection, client);
return supported;
};
export class TransactionUnavailableError extends Error {
@ -90,7 +110,7 @@ export class TransactionUnavailableError extends Error {
const requiredTransactionSupport = new WeakMap<
Connection,
{
client: ReturnType<Connection['getClient']>;
client: TransactionClient;
supported: boolean | null;
}
>();
@ -108,8 +128,12 @@ export async function startRequiredTransactionSession(
}
// Assignment happens only on a confirmed verdict; operational failures remain retryable.
state.supported = await getTransactionSupport(mongoose, state.supported);
assertTransactionClient(mongoose, connection, client);
if (!state.supported) throw new TransactionUnavailableError();
return mongoose.startSession({ defaultTransactionOptions: { readPreference: 'primary' } });
// The driver's synchronous call binds the session to the client we just checked.
return client.startSession({
defaultTransactionOptions: { readPreference: 'primary' },
});
}
/** Prepare indexes outside the transaction, after confirming storage capability. */