mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 21:08:17 +00:00
🔧 fix: Fence Channel Startup and Transaction Capability (#17831)
* 🔧 fix: Fence Channel Startup and Transaction Capability * 🔧 fix: Gate Clustered Workers Until Startup Completes
This commit is contained in:
parent
1f27f5848a
commit
fdf75e0174
10 changed files with 227 additions and 16 deletions
32
.github/scripts/test-channel-compose.mjs
vendored
32
.github/scripts/test-channel-compose.mjs
vendored
|
|
@ -1,9 +1,11 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { execFile, execFileSync } from 'node:child_process';
|
||||
import { copyFileSync, mkdtempSync, rmSync } from 'node:fs';
|
||||
import { createServer } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const root = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const fixture = mkdtempSync(join(tmpdir(), 'channel-compose-'));
|
||||
|
|
@ -48,6 +50,7 @@ function configuration(tag, source = false, shared = false, gatewayPort) {
|
|||
...process.env,
|
||||
UID: '1000',
|
||||
GID: '1000',
|
||||
PORT: '3080',
|
||||
LIBRECHAT_IMAGE_TAG: tag,
|
||||
SLACK_APP_ID: 'A123',
|
||||
SLACK_CLIENT_ID: '123.456',
|
||||
|
|
@ -121,3 +124,30 @@ assert.equal(
|
|||
'3999',
|
||||
);
|
||||
console.log('PASS: shared Compose needs no app credentials and clears all bound-gateway settings');
|
||||
|
||||
// Run the shipped healthcheck against an API that listens before initialization completes.
|
||||
let ready = false;
|
||||
const server = createServer((req, res) => {
|
||||
res.writeHead(req.url === '/health' || (req.url === '/readyz' && ready) ? 200 : 503);
|
||||
res.end();
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const [kind, command, flag, script] = configuration('').api.healthcheck.test;
|
||||
assert.equal(kind, 'CMD');
|
||||
assert.equal(command, 'node');
|
||||
assert.equal(flag, '-e');
|
||||
const probe = () =>
|
||||
promisify(execFile)(process.execPath, [
|
||||
flag,
|
||||
script.replace('127.0.0.1:3080', `127.0.0.1:${port}`),
|
||||
]);
|
||||
assert.equal((await fetch(`http://127.0.0.1:${port}/health`)).status, 200);
|
||||
await assert.rejects(probe(), { code: 1 });
|
||||
ready = true;
|
||||
await probe();
|
||||
console.log('PASS: gateway dependency stays unhealthy until API readiness succeeds');
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
}
|
||||
|
|
|
|||
4
.github/scripts/test-channel-release.mjs
vendored
4
.github/scripts/test-channel-release.mjs
vendored
|
|
@ -8,6 +8,10 @@ import yaml from 'js-yaml';
|
|||
const readWorkflow = (file) =>
|
||||
yaml.load(readFileSync(new URL(`../workflows/${file}`, import.meta.url), 'utf8'));
|
||||
const manual = readWorkflow('main-image-workflow.yml');
|
||||
assert.ok(
|
||||
readWorkflow('retry-docker-builds.yml').on.workflow_run.workflows.includes(manual.name),
|
||||
'the retry workflow must subscribe to the manual release publisher by its current name',
|
||||
);
|
||||
for (const workflow of [readWorkflow('tag-images.yml')]) {
|
||||
const images = JSON.parse(workflow.jobs.publish.with.images);
|
||||
const gateway = images.find((image) => image.image_name === 'librechat-channel-gateway');
|
||||
|
|
|
|||
2
.github/workflows/channel-gateway.yml
vendored
2
.github/workflows/channel-gateway.yml
vendored
|
|
@ -17,6 +17,7 @@ on:
|
|||
- '.github/scripts/test-channel-release.mjs'
|
||||
- '.github/workflows/tag-images.yml'
|
||||
- '.github/workflows/main-image-workflow.yml'
|
||||
- '.github/workflows/retry-docker-builds.yml'
|
||||
- '.github/workflows/docker-publish.yml'
|
||||
- 'helm/librechat/**'
|
||||
- 'docker-compose.mongo-replica.yml'
|
||||
|
|
@ -40,6 +41,7 @@ on:
|
|||
- '.github/scripts/test-channel-release.mjs'
|
||||
- '.github/workflows/tag-images.yml'
|
||||
- '.github/workflows/main-image-workflow.yml'
|
||||
- '.github/workflows/retry-docker-builds.yml'
|
||||
- '.github/workflows/docker-publish.yml'
|
||||
- 'helm/librechat/**'
|
||||
- 'docker-compose.mongo-replica.yml'
|
||||
|
|
|
|||
2
.github/workflows/retry-docker-builds.yml
vendored
2
.github/workflows/retry-docker-builds.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
workflow_run:
|
||||
workflows:
|
||||
- Docker Build Smoke Tests
|
||||
- Docker Compose Build Latest Main Image Tag (Manual Dispatch)
|
||||
- Publish Latest Stable Release Images (Manual Dispatch)
|
||||
- Docker Dev Branch Images Build
|
||||
- Docker Dev Images Build
|
||||
- Docker Dev Staging Images Build
|
||||
|
|
|
|||
|
|
@ -59,6 +59,9 @@ const {
|
|||
cacheConfig,
|
||||
createClusteredFileSweep,
|
||||
createRequestSanitizer,
|
||||
createStartupReadiness,
|
||||
keyvRedisClient,
|
||||
ioredisClient,
|
||||
} = require('@librechat/api');
|
||||
const { connectDb, indexSync } = require('~/db');
|
||||
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
|
||||
|
|
@ -379,6 +382,11 @@ if (cluster.isMaster) {
|
|||
* Each worker runs a full Express server instance
|
||||
*/
|
||||
const app = express();
|
||||
const readiness = createStartupReadiness({
|
||||
useRedis: cacheConfig.USE_REDIS,
|
||||
ioredis: ioredisClient,
|
||||
keyv: keyvRedisClient,
|
||||
});
|
||||
app.locals.codeApiUploadRegistry = createCodeApiUploadRegistry();
|
||||
// The clustered entrypoint deliberately does not arm the v1 schedule engine,
|
||||
// but an already-fired scheduled generation can still reach HITL here. Settle
|
||||
|
|
@ -607,6 +615,8 @@ if (cluster.isMaster) {
|
|||
|
||||
/** Health check endpoint */
|
||||
app.get('/health', (_req, res) => res.status(200).send('OK'));
|
||||
app.get('/readyz', readiness.handler);
|
||||
app.use('/api', readiness.gate);
|
||||
|
||||
/** Middleware */
|
||||
app.use(requestContextMiddleware);
|
||||
|
|
@ -764,6 +774,7 @@ if (cluster.isMaster) {
|
|||
registerShutdownTask('channel feedback export', stopChannelFeedback, { priority: 100 });
|
||||
registerShutdownTask('channel authority cleanup', stopChannelCleanup, { priority: 100 });
|
||||
await initializeChannelCleanup();
|
||||
readiness.markReady();
|
||||
} catch (initErr) {
|
||||
logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);
|
||||
process.exit(1);
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ services:
|
|||
'CMD',
|
||||
'node',
|
||||
'-e',
|
||||
"fetch('http://127.0.0.1:${PORT:-3080}/health',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})",
|
||||
"fetch('http://127.0.0.1:${PORT:-3080}/readyz',{signal:AbortSignal.timeout(4000)}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>{process.exitCode=1})",
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import IoRedis from 'ioredis';
|
||||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import { createClient } from '@keyv/redis';
|
||||
import type { RedisClientType } from '@redis/client';
|
||||
import type { Request, Response } from 'express';
|
||||
import type { RespServer } from '~/cache/__tests__/resp.helper';
|
||||
import type { RedisReadinessClients } from './readiness';
|
||||
import { createReadinessHandler, createStartupReadiness, isRedisReady } from './readiness';
|
||||
import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper';
|
||||
import { createReadinessHandler, isRedisReady } from './readiness';
|
||||
|
||||
const clients = (overrides: Partial<RedisReadinessClients> = {}): RedisReadinessClients => ({
|
||||
useRedis: true,
|
||||
|
|
@ -129,3 +131,51 @@ describe('createReadinessHandler', () => {
|
|||
expect(respond(handler).code).toBe(200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('worker startup readiness', () => {
|
||||
const worker = (redis: RedisReadinessClients = clients({ useRedis: false })) => {
|
||||
const readiness = createStartupReadiness(redis);
|
||||
const app = express();
|
||||
app.get('/readyz', readiness.handler);
|
||||
app.use('/api', readiness.gate);
|
||||
app.get('/api/work', (_req, res) => {
|
||||
res.sendStatus(200);
|
||||
});
|
||||
return { app, readiness };
|
||||
};
|
||||
|
||||
it('refuses API work until post-listen initialization completes', async () => {
|
||||
const { app, readiness } = worker();
|
||||
await request(app).get('/readyz').expect(503);
|
||||
await request(app)
|
||||
.get('/api/work')
|
||||
.expect('Retry-After', '1')
|
||||
.expect(503)
|
||||
.expect(({ body }) => expect(body.code).toBe('SERVER_NOT_READY'));
|
||||
readiness.markReady();
|
||||
await request(app).get('/readyz').expect(200);
|
||||
await request(app).get('/api/work').expect(200);
|
||||
});
|
||||
|
||||
it('keeps another starting worker closed when its sibling becomes ready', async () => {
|
||||
const first = worker();
|
||||
const second = worker();
|
||||
first.readiness.markReady();
|
||||
await request(first.app).get('/api/work').expect(200);
|
||||
expect((await request(second.app).get('/readyz')).status).toBe(503);
|
||||
await request(second.app).get('/api/work').expect(503);
|
||||
});
|
||||
|
||||
it('rechecks Redis readiness for both the probe and API admission', async () => {
|
||||
const keyv = { isReady: true };
|
||||
const { app, readiness } = worker(clients({ keyv }));
|
||||
readiness.markReady();
|
||||
await request(app).get('/api/work').expect(200);
|
||||
keyv.isReady = false;
|
||||
expect((await request(app).get('/readyz')).status).toBe(503);
|
||||
await request(app).get('/api/work').expect(503);
|
||||
keyv.isReady = true;
|
||||
await request(app).get('/readyz').expect(200);
|
||||
await request(app).get('/api/work').expect(200);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { Request, Response, RequestHandler } from 'express';
|
||||
import type { RedisClientType } from '@redis/client';
|
||||
import type { Request, Response } from 'express';
|
||||
import type { Redis, Cluster } from 'ioredis';
|
||||
|
||||
type NodeRedisReadiness = Pick<RedisClientType, 'isReady'>;
|
||||
|
|
@ -63,3 +63,30 @@ export const createReadinessHandler =
|
|||
}
|
||||
res.status(200).send('OK');
|
||||
};
|
||||
|
||||
/** Per-worker readiness: a healthy sibling must not admit work to an initializing worker. */
|
||||
export function createStartupReadiness(redis: RedisReadinessClients): {
|
||||
handler: RequestHandler;
|
||||
gate: RequestHandler;
|
||||
markReady: () => void;
|
||||
} {
|
||||
let initialized = false;
|
||||
const gate: RequestHandler = (_req, res, next) => {
|
||||
if (initialized && isRedisReady(redis)) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
res.set('Retry-After', '1');
|
||||
res.status(503).json({
|
||||
code: 'SERVER_NOT_READY',
|
||||
error: 'Server is not ready. Please retry shortly.',
|
||||
});
|
||||
};
|
||||
return {
|
||||
handler: createReadinessHandler(() => initialized, redis),
|
||||
gate,
|
||||
markReady: () => {
|
||||
initialized = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
supportsTransactions,
|
||||
startRequiredTransactionSession,
|
||||
TransactionUnavailableError,
|
||||
withRequiredTransaction,
|
||||
} from './transactions';
|
||||
|
||||
jest.mock('~/config/winston', () => ({ debug: jest.fn() }));
|
||||
|
|
@ -140,7 +141,7 @@ describe('transaction capability against real MongoDB', () => {
|
|||
it('reuses a required-session verdict instead of probing every subsequent transition', async () => {
|
||||
const db = new mongoose.Mongoose();
|
||||
await db.connect(repl.getUri(), { readPreference: 'secondaryPreferred' });
|
||||
const starts = jest.spyOn(db, 'startSession');
|
||||
const starts = jest.spyOn(db.connection.getClient(), 'startSession');
|
||||
try {
|
||||
const first = await startRequiredTransactionSession(db);
|
||||
await first.endSession();
|
||||
|
|
@ -164,7 +165,7 @@ describe('transaction capability against real MongoDB', () => {
|
|||
await expect(startRequiredTransactionSession(db)).rejects.toBe(failure);
|
||||
const session = await startRequiredTransactionSession(db);
|
||||
await session.endSession();
|
||||
expect(starts).toHaveBeenCalledTimes(3);
|
||||
expect(starts).toHaveBeenCalledTimes(2); // Failed probe, then successful probe.
|
||||
} finally {
|
||||
await db.disconnect();
|
||||
}
|
||||
|
|
@ -196,4 +197,66 @@ describe('transaction capability against real MongoDB', () => {
|
|||
);
|
||||
expect(starts).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('rejects an old pending verdict and probes the replacement client before private effects', async () => {
|
||||
const db = new mongoose.Mongoose();
|
||||
await db.connect(repl.getUri());
|
||||
const probeSession = await db.startSession();
|
||||
const commit = probeSession.commitTransaction.bind(probeSession);
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const paused = new Promise<void>((resolve) => (release = resolve));
|
||||
const committed = new Promise<void>((resolve) => (entered = resolve));
|
||||
jest.spyOn(probeSession, 'commitTransaction').mockImplementationOnce(async () => {
|
||||
await commit();
|
||||
entered();
|
||||
await paused;
|
||||
});
|
||||
const starts = jest.spyOn(db, 'startSession').mockResolvedValueOnce(probeSession);
|
||||
const prepare = jest.fn(async () => undefined);
|
||||
const write = jest.fn(async () => 'private-effect');
|
||||
try {
|
||||
const stale = withRequiredTransaction(db, write, prepare);
|
||||
await committed;
|
||||
await db.disconnect();
|
||||
await db.connect(mongo.getUri());
|
||||
const current = withRequiredTransaction(db, write, prepare);
|
||||
const outcomes = Promise.allSettled([stale, current]);
|
||||
release();
|
||||
expect(await outcomes).toEqual([
|
||||
{ status: 'rejected', reason: expect.any(db.mongo.MongoNetworkError) },
|
||||
{ status: 'rejected', reason: expect.any(TransactionUnavailableError) },
|
||||
]);
|
||||
await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf(
|
||||
TransactionUnavailableError,
|
||||
);
|
||||
expect(starts).toHaveBeenCalledTimes(2); // The old probe and a fresh standalone probe.
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
expect(write).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release();
|
||||
await db.disconnect();
|
||||
}
|
||||
});
|
||||
|
||||
it('rechecks the client after awaiting a cached capability verdict', async () => {
|
||||
const db = new mongoose.Mongoose();
|
||||
await db.connect(repl.getUri());
|
||||
try {
|
||||
const session = await startRequiredTransactionSession(db);
|
||||
await session.endSession();
|
||||
const client = db.connection.getClient();
|
||||
jest
|
||||
.spyOn(db.connection, 'getClient')
|
||||
.mockReturnValueOnce(client)
|
||||
.mockReturnValueOnce(standalone.connection.getClient());
|
||||
const starts = jest.spyOn(db, 'startSession');
|
||||
await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf(
|
||||
db.mongo.MongoNetworkError,
|
||||
);
|
||||
expect(starts).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await db.disconnect();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -62,21 +62,41 @@ export const supportsTransactions = async (
|
|||
}
|
||||
};
|
||||
|
||||
const inflightProbes = new WeakMap<import('mongoose').Connection, Promise<boolean>>();
|
||||
type TransactionClient = ReturnType<Connection['getClient']>;
|
||||
const inflightProbes = new WeakMap<Connection, WeakMap<TransactionClient, Promise<boolean>>>();
|
||||
|
||||
/** Share first probes per connection. Callers cache only a successful capability verdict. */
|
||||
function assertTransactionClient(
|
||||
mongoose: typeof import('mongoose'),
|
||||
connection: Connection,
|
||||
client: TransactionClient,
|
||||
): void {
|
||||
if (mongoose.connection !== connection || connection.getClient() !== client) {
|
||||
throw new mongoose.mongo.MongoNetworkError('MongoDB client changed during transaction setup');
|
||||
}
|
||||
}
|
||||
|
||||
/** Share first probes only for the same connection and client. */
|
||||
export const getTransactionSupport = async (
|
||||
mongoose: typeof import('mongoose'),
|
||||
transactionSupportCache: boolean | null,
|
||||
): Promise<boolean> => {
|
||||
if (transactionSupportCache !== null) return transactionSupportCache;
|
||||
const connection = mongoose.connection;
|
||||
let probe = inflightProbes.get(connection);
|
||||
if (!probe) {
|
||||
probe = supportsTransactions(mongoose).finally(() => inflightProbes.delete(connection));
|
||||
inflightProbes.set(connection, probe);
|
||||
const client = connection.getClient();
|
||||
let probes = inflightProbes.get(connection);
|
||||
if (!probes) {
|
||||
probes = new WeakMap();
|
||||
inflightProbes.set(connection, probes);
|
||||
}
|
||||
return probe;
|
||||
let probe = probes.get(client);
|
||||
if (!probe) {
|
||||
const pending = probes;
|
||||
probe = supportsTransactions(mongoose).finally(() => pending.delete(client));
|
||||
probes.set(client, probe);
|
||||
}
|
||||
const supported = await probe;
|
||||
assertTransactionClient(mongoose, connection, client);
|
||||
return supported;
|
||||
};
|
||||
|
||||
export class TransactionUnavailableError extends Error {
|
||||
|
|
@ -90,7 +110,7 @@ export class TransactionUnavailableError extends Error {
|
|||
const requiredTransactionSupport = new WeakMap<
|
||||
Connection,
|
||||
{
|
||||
client: ReturnType<Connection['getClient']>;
|
||||
client: TransactionClient;
|
||||
supported: boolean | null;
|
||||
}
|
||||
>();
|
||||
|
|
@ -108,8 +128,12 @@ export async function startRequiredTransactionSession(
|
|||
}
|
||||
// Assignment happens only on a confirmed verdict; operational failures remain retryable.
|
||||
state.supported = await getTransactionSupport(mongoose, state.supported);
|
||||
assertTransactionClient(mongoose, connection, client);
|
||||
if (!state.supported) throw new TransactionUnavailableError();
|
||||
return mongoose.startSession({ defaultTransactionOptions: { readPreference: 'primary' } });
|
||||
// The driver's synchronous call binds the session to the client we just checked.
|
||||
return client.startSession({
|
||||
defaultTransactionOptions: { readPreference: 'primary' },
|
||||
});
|
||||
}
|
||||
|
||||
/** Prepare indexes outside the transaction, after confirming storage capability. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue