From fdf75e01745ffc6cdcbc6709e1a72b0aa88a4152 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:04:10 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20fix:=20Fence=20Channel=20Startup?= =?UTF-8?q?=20and=20Transaction=20Capability=20(#17831)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔧 fix: Fence Channel Startup and Transaction Capability * 🔧 fix: Gate Clustered Workers Until Startup Completes --- .github/scripts/test-channel-compose.mjs | 32 ++++++++- .github/scripts/test-channel-release.mjs | 4 ++ .github/workflows/channel-gateway.yml | 2 + .github/workflows/retry-docker-builds.yml | 2 +- api/server/experimental.js | 11 +++ docker-compose.channels.yml | 2 +- packages/api/src/app/readiness.spec.ts | 52 +++++++++++++- packages/api/src/app/readiness.ts | 29 +++++++- .../src/utils/transactions.spec.ts | 67 ++++++++++++++++++- .../data-schemas/src/utils/transactions.ts | 42 +++++++++--- 10 files changed, 227 insertions(+), 16 deletions(-) diff --git a/.github/scripts/test-channel-compose.mjs b/.github/scripts/test-channel-compose.mjs index 54e13306a2..7126cb189c 100644 --- a/.github/scripts/test-channel-compose.mjs +++ b/.github/scripts/test-channel-compose.mjs @@ -1,9 +1,11 @@ import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; +import { execFile, execFileSync } from 'node:child_process'; import { copyFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; const root = fileURLToPath(new URL('../..', import.meta.url)); const fixture = mkdtempSync(join(tmpdir(), 'channel-compose-')); @@ -48,6 +50,7 @@ function configuration(tag, source = false, shared = false, gatewayPort) { ...process.env, UID: '1000', GID: '1000', + PORT: '3080', LIBRECHAT_IMAGE_TAG: tag, SLACK_APP_ID: 'A123', SLACK_CLIENT_ID: '123.456', @@ -121,3 +124,30 @@ assert.equal( '3999', ); console.log('PASS: shared Compose needs no app credentials and clears all bound-gateway settings'); + +// Run the shipped healthcheck against an API that listens before initialization completes. +let ready = false; +const server = createServer((req, res) => { + res.writeHead(req.url === '/health' || (req.url === '/readyz' && ready) ? 200 : 503); + res.end(); +}); +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); +try { + const { port } = server.address(); + const [kind, command, flag, script] = configuration('').api.healthcheck.test; + assert.equal(kind, 'CMD'); + assert.equal(command, 'node'); + assert.equal(flag, '-e'); + const probe = () => + promisify(execFile)(process.execPath, [ + flag, + script.replace('127.0.0.1:3080', `127.0.0.1:${port}`), + ]); + assert.equal((await fetch(`http://127.0.0.1:${port}/health`)).status, 200); + await assert.rejects(probe(), { code: 1 }); + ready = true; + await probe(); + console.log('PASS: gateway dependency stays unhealthy until API readiness succeeds'); +} finally { + await new Promise((resolve) => server.close(resolve)); +} diff --git a/.github/scripts/test-channel-release.mjs b/.github/scripts/test-channel-release.mjs index d233a6b40f..7db9ef66bf 100644 --- a/.github/scripts/test-channel-release.mjs +++ b/.github/scripts/test-channel-release.mjs @@ -8,6 +8,10 @@ import yaml from 'js-yaml'; const readWorkflow = (file) => yaml.load(readFileSync(new URL(`../workflows/${file}`, import.meta.url), 'utf8')); const manual = readWorkflow('main-image-workflow.yml'); +assert.ok( + readWorkflow('retry-docker-builds.yml').on.workflow_run.workflows.includes(manual.name), + 'the retry workflow must subscribe to the manual release publisher by its current name', +); for (const workflow of [readWorkflow('tag-images.yml')]) { const images = JSON.parse(workflow.jobs.publish.with.images); const gateway = images.find((image) => image.image_name === 'librechat-channel-gateway'); diff --git a/.github/workflows/channel-gateway.yml b/.github/workflows/channel-gateway.yml index 774009fe83..483c93c1f3 100644 --- a/.github/workflows/channel-gateway.yml +++ b/.github/workflows/channel-gateway.yml @@ -17,6 +17,7 @@ on: - '.github/scripts/test-channel-release.mjs' - '.github/workflows/tag-images.yml' - '.github/workflows/main-image-workflow.yml' + - '.github/workflows/retry-docker-builds.yml' - '.github/workflows/docker-publish.yml' - 'helm/librechat/**' - 'docker-compose.mongo-replica.yml' @@ -40,6 +41,7 @@ on: - '.github/scripts/test-channel-release.mjs' - '.github/workflows/tag-images.yml' - '.github/workflows/main-image-workflow.yml' + - '.github/workflows/retry-docker-builds.yml' - '.github/workflows/docker-publish.yml' - 'helm/librechat/**' - 'docker-compose.mongo-replica.yml' diff --git a/.github/workflows/retry-docker-builds.yml b/.github/workflows/retry-docker-builds.yml index f454397719..88bc4e59f3 100644 --- a/.github/workflows/retry-docker-builds.yml +++ b/.github/workflows/retry-docker-builds.yml @@ -4,7 +4,7 @@ on: workflow_run: workflows: - Docker Build Smoke Tests - - Docker Compose Build Latest Main Image Tag (Manual Dispatch) + - Publish Latest Stable Release Images (Manual Dispatch) - Docker Dev Branch Images Build - Docker Dev Images Build - Docker Dev Staging Images Build diff --git a/api/server/experimental.js b/api/server/experimental.js index f252578caf..bc4b1b1103 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -59,6 +59,9 @@ const { cacheConfig, createClusteredFileSweep, createRequestSanitizer, + createStartupReadiness, + keyvRedisClient, + ioredisClient, } = require('@librechat/api'); const { connectDb, indexSync } = require('~/db'); const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); @@ -379,6 +382,11 @@ if (cluster.isMaster) { * Each worker runs a full Express server instance */ const app = express(); + const readiness = createStartupReadiness({ + useRedis: cacheConfig.USE_REDIS, + ioredis: ioredisClient, + keyv: keyvRedisClient, + }); app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry(); // The clustered entrypoint deliberately does not arm the v1 schedule engine, // but an already-fired scheduled generation can still reach HITL here. Settle @@ -607,6 +615,8 @@ if (cluster.isMaster) { /** Health check endpoint */ app.get('/health', (_req, res) => res.status(200).send('OK')); + app.get('/readyz', readiness.handler); + app.use('/api', readiness.gate); /** Middleware */ app.use(requestContextMiddleware); @@ -764,6 +774,7 @@ if (cluster.isMaster) { registerShutdownTask('channel feedback export', stopChannelFeedback, { priority: 100 }); registerShutdownTask('channel authority cleanup', stopChannelCleanup, { priority: 100 }); await initializeChannelCleanup(); + readiness.markReady(); } catch (initErr) { logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr); process.exit(1); diff --git a/docker-compose.channels.yml b/docker-compose.channels.yml index d9ce6ed1d5..515af33aa9 100644 --- a/docker-compose.channels.yml +++ b/docker-compose.channels.yml @@ -11,7 +11,7 @@ services: 'CMD', 'node', '-e', - "fetch('http://127.0.0.1:${PORT:-3080}/health',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})", + "fetch('http://127.0.0.1:${PORT:-3080}/readyz',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})", ] interval: 10s timeout: 5s diff --git a/packages/api/src/app/readiness.spec.ts b/packages/api/src/app/readiness.spec.ts index dbed6cc2d8..380a12b862 100644 --- a/packages/api/src/app/readiness.spec.ts +++ b/packages/api/src/app/readiness.spec.ts @@ -1,11 +1,13 @@ import IoRedis from 'ioredis'; +import express from 'express'; +import request from 'supertest'; import { createClient } from '@keyv/redis'; import type { RedisClientType } from '@redis/client'; import type { Request, Response } from 'express'; import type { RespServer } from '~/cache/__tests__/resp.helper'; import type { RedisReadinessClients } from './readiness'; +import { createReadinessHandler, createStartupReadiness, isRedisReady } from './readiness'; import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper'; -import { createReadinessHandler, isRedisReady } from './readiness'; const clients = (overrides: Partial = {}): RedisReadinessClients => ({ useRedis: true, @@ -129,3 +131,51 @@ describe('createReadinessHandler', () => { expect(respond(handler).code).toBe(200); }); }); + +describe('worker startup readiness', () => { + const worker = (redis: RedisReadinessClients = clients({ useRedis: false })) => { + const readiness = createStartupReadiness(redis); + const app = express(); + app.get('/readyz', readiness.handler); + app.use('/api', readiness.gate); + app.get('/api/work', (_req, res) => { + res.sendStatus(200); + }); + return { app, readiness }; + }; + + it('refuses API work until post-listen initialization completes', async () => { + const { app, readiness } = worker(); + await request(app).get('/readyz').expect(503); + await request(app) + .get('/api/work') + .expect('Retry-After', '1') + .expect(503) + .expect(({ body }) => expect(body.code).toBe('SERVER_NOT_READY')); + readiness.markReady(); + await request(app).get('/readyz').expect(200); + await request(app).get('/api/work').expect(200); + }); + + it('keeps another starting worker closed when its sibling becomes ready', async () => { + const first = worker(); + const second = worker(); + first.readiness.markReady(); + await request(first.app).get('/api/work').expect(200); + expect((await request(second.app).get('/readyz')).status).toBe(503); + await request(second.app).get('/api/work').expect(503); + }); + + it('rechecks Redis readiness for both the probe and API admission', async () => { + const keyv = { isReady: true }; + const { app, readiness } = worker(clients({ keyv })); + readiness.markReady(); + await request(app).get('/api/work').expect(200); + keyv.isReady = false; + expect((await request(app).get('/readyz')).status).toBe(503); + await request(app).get('/api/work').expect(503); + keyv.isReady = true; + await request(app).get('/readyz').expect(200); + await request(app).get('/api/work').expect(200); + }); +}); diff --git a/packages/api/src/app/readiness.ts b/packages/api/src/app/readiness.ts index 29a0ed4f5a..356b99f52b 100644 --- a/packages/api/src/app/readiness.ts +++ b/packages/api/src/app/readiness.ts @@ -1,5 +1,5 @@ +import type { Request, Response, RequestHandler } from 'express'; import type { RedisClientType } from '@redis/client'; -import type { Request, Response } from 'express'; import type { Redis, Cluster } from 'ioredis'; type NodeRedisReadiness = Pick; @@ -63,3 +63,30 @@ export const createReadinessHandler = } res.status(200).send('OK'); }; + +/** Per-worker readiness: a healthy sibling must not admit work to an initializing worker. */ +export function createStartupReadiness(redis: RedisReadinessClients): { + handler: RequestHandler; + gate: RequestHandler; + markReady: () => void; +} { + let initialized = false; + const gate: RequestHandler = (_req, res, next) => { + if (initialized && isRedisReady(redis)) { + next(); + return; + } + res.set('Retry-After', '1'); + res.status(503).json({ + code: 'SERVER_NOT_READY', + error: 'Server is not ready. Please retry shortly.', + }); + }; + return { + handler: createReadinessHandler(() => initialized, redis), + gate, + markReady: () => { + initialized = true; + }, + }; +} diff --git a/packages/data-schemas/src/utils/transactions.spec.ts b/packages/data-schemas/src/utils/transactions.spec.ts index 1126b6c2f1..2eabd1cd9c 100644 --- a/packages/data-schemas/src/utils/transactions.spec.ts +++ b/packages/data-schemas/src/utils/transactions.spec.ts @@ -5,6 +5,7 @@ import { supportsTransactions, startRequiredTransactionSession, TransactionUnavailableError, + withRequiredTransaction, } from './transactions'; jest.mock('~/config/winston', () => ({ debug: jest.fn() })); @@ -140,7 +141,7 @@ describe('transaction capability against real MongoDB', () => { it('reuses a required-session verdict instead of probing every subsequent transition', async () => { const db = new mongoose.Mongoose(); await db.connect(repl.getUri(), { readPreference: 'secondaryPreferred' }); - const starts = jest.spyOn(db, 'startSession'); + const starts = jest.spyOn(db.connection.getClient(), 'startSession'); try { const first = await startRequiredTransactionSession(db); await first.endSession(); @@ -164,7 +165,7 @@ describe('transaction capability against real MongoDB', () => { await expect(startRequiredTransactionSession(db)).rejects.toBe(failure); const session = await startRequiredTransactionSession(db); await session.endSession(); - expect(starts).toHaveBeenCalledTimes(3); + expect(starts).toHaveBeenCalledTimes(2); // Failed probe, then successful probe. } finally { await db.disconnect(); } @@ -196,4 +197,66 @@ describe('transaction capability against real MongoDB', () => { ); expect(starts).toHaveBeenCalledTimes(1); }); + + it('rejects an old pending verdict and probes the replacement client before private effects', async () => { + const db = new mongoose.Mongoose(); + await db.connect(repl.getUri()); + const probeSession = await db.startSession(); + const commit = probeSession.commitTransaction.bind(probeSession); + let release!: () => void; + let entered!: () => void; + const paused = new Promise((resolve) => (release = resolve)); + const committed = new Promise((resolve) => (entered = resolve)); + jest.spyOn(probeSession, 'commitTransaction').mockImplementationOnce(async () => { + await commit(); + entered(); + await paused; + }); + const starts = jest.spyOn(db, 'startSession').mockResolvedValueOnce(probeSession); + const prepare = jest.fn(async () => undefined); + const write = jest.fn(async () => 'private-effect'); + try { + const stale = withRequiredTransaction(db, write, prepare); + await committed; + await db.disconnect(); + await db.connect(mongo.getUri()); + const current = withRequiredTransaction(db, write, prepare); + const outcomes = Promise.allSettled([stale, current]); + release(); + expect(await outcomes).toEqual([ + { status: 'rejected', reason: expect.any(db.mongo.MongoNetworkError) }, + { status: 'rejected', reason: expect.any(TransactionUnavailableError) }, + ]); + await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf( + TransactionUnavailableError, + ); + expect(starts).toHaveBeenCalledTimes(2); // The old probe and a fresh standalone probe. + expect(prepare).not.toHaveBeenCalled(); + expect(write).not.toHaveBeenCalled(); + } finally { + release(); + await db.disconnect(); + } + }); + + it('rechecks the client after awaiting a cached capability verdict', async () => { + const db = new mongoose.Mongoose(); + await db.connect(repl.getUri()); + try { + const session = await startRequiredTransactionSession(db); + await session.endSession(); + const client = db.connection.getClient(); + jest + .spyOn(db.connection, 'getClient') + .mockReturnValueOnce(client) + .mockReturnValueOnce(standalone.connection.getClient()); + const starts = jest.spyOn(db, 'startSession'); + await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf( + db.mongo.MongoNetworkError, + ); + expect(starts).not.toHaveBeenCalled(); + } finally { + await db.disconnect(); + } + }); }); diff --git a/packages/data-schemas/src/utils/transactions.ts b/packages/data-schemas/src/utils/transactions.ts index a1e8e9a51d..bcb3a2d631 100644 --- a/packages/data-schemas/src/utils/transactions.ts +++ b/packages/data-schemas/src/utils/transactions.ts @@ -62,21 +62,41 @@ export const supportsTransactions = async ( } }; -const inflightProbes = new WeakMap>(); +type TransactionClient = ReturnType; +const inflightProbes = new WeakMap>>(); -/** Share first probes per connection. Callers cache only a successful capability verdict. */ +function assertTransactionClient( + mongoose: typeof import('mongoose'), + connection: Connection, + client: TransactionClient, +): void { + if (mongoose.connection !== connection || connection.getClient() !== client) { + throw new mongoose.mongo.MongoNetworkError('MongoDB client changed during transaction setup'); + } +} + +/** Share first probes only for the same connection and client. */ export const getTransactionSupport = async ( mongoose: typeof import('mongoose'), transactionSupportCache: boolean | null, ): Promise => { if (transactionSupportCache !== null) return transactionSupportCache; const connection = mongoose.connection; - let probe = inflightProbes.get(connection); - if (!probe) { - probe = supportsTransactions(mongoose).finally(() => inflightProbes.delete(connection)); - inflightProbes.set(connection, probe); + const client = connection.getClient(); + let probes = inflightProbes.get(connection); + if (!probes) { + probes = new WeakMap(); + inflightProbes.set(connection, probes); } - return probe; + let probe = probes.get(client); + if (!probe) { + const pending = probes; + probe = supportsTransactions(mongoose).finally(() => pending.delete(client)); + probes.set(client, probe); + } + const supported = await probe; + assertTransactionClient(mongoose, connection, client); + return supported; }; export class TransactionUnavailableError extends Error { @@ -90,7 +110,7 @@ export class TransactionUnavailableError extends Error { const requiredTransactionSupport = new WeakMap< Connection, { - client: ReturnType; + client: TransactionClient; supported: boolean | null; } >(); @@ -108,8 +128,12 @@ export async function startRequiredTransactionSession( } // Assignment happens only on a confirmed verdict; operational failures remain retryable. state.supported = await getTransactionSupport(mongoose, state.supported); + assertTransactionClient(mongoose, connection, client); if (!state.supported) throw new TransactionUnavailableError(); - return mongoose.startSession({ defaultTransactionOptions: { readPreference: 'primary' } }); + // The driver's synchronous call binds the session to the client we just checked. + return client.startSession({ + defaultTransactionOptions: { readPreference: 'primary' }, + }); } /** Prepare indexes outside the transaction, after confirming storage capability. */