🔕 fix: Quiet Client Disconnects, Fall Back When Hash Workers Fail, Log Speech Truncation (#17278)

This commit is contained in:
Marco Beretta 2026-10-20 10:47:15 +02:00 • committed by GitHub
parent 6c0d3a7cbb
commit ded1e734e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 243 additions and 21 deletions

View file

@ -1,5 +1,6 @@
import path from 'node:path';
import { hashSync } from 'bcryptjs';
import { logger } from '@librechat/data-schemas';
import { createHashPool } from './hashing';
const workerPath = path.join(
@ -39,13 +40,35 @@ describe('bcrypt hash pool', () => {
expect(worstLagMs).toBeLessThan(50);
});
it('rejects a job whose worker dies instead of leaving it pending', async () => {
it('finishes a job on the main thread when its worker dies, warning once', async () => {
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const crashing = createHashPool(
path.join(__dirname, '__fixtures__', 'crashing-hash-worker.cjs'),
1,
);
try {
const hash = await crashing.hash('secret', 4);
await expect(crashing.hash('secret', 4)).rejects.toThrow('Password hashing worker stopped');
await crashing.close();
await expect(crashing.compare('secret', hash)).resolves.toBe(true);
await expect(crashing.compare('other', hash)).resolves.toBe(false);
expect(warn).toHaveBeenCalledTimes(1);
} finally {
warn.mockRestore();
await crashing.close();
}
});
it('hashes on the main thread when the worker cannot start', async () => {
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const missing = createHashPool(path.join(__dirname, '__fixtures__', 'missing-worker.cjs'), 1);
try {
const hash = await missing.hash('secret', 4);
await expect(missing.compare('secret', hash)).resolves.toBe(true);
expect(warn).toHaveBeenCalledTimes(1);
} finally {
warn.mockRestore();
await missing.close();
}
});
});

View file

@ -1,6 +1,8 @@
import path from 'node:path';
import { compare, hash } from 'bcryptjs';
import { Worker } from 'node:worker_threads';
import { availableParallelism } from 'node:os';
import { logger } from '@librechat/data-schemas';
import type { HashJob, HashReply } from './hash-worker';
type HashRequest =
@ -15,6 +17,21 @@ interface Pending {
interface HashWorker {
worker: Worker;
pending: Map<number, Pending>;
/** Whether this worker ever answered; one that dies first means workers cannot start here. */
answered: boolean;
}
class HashWorkerStoppedError extends Error {
constructor() {
super('Password hashing worker stopped');
}
}
/** The main-thread path: slower under load, but sign-in keeps working without workers. */
function runInProcess(request: HashRequest): Promise<string | boolean> {
return request.op === 'hash'
? hash(request.plain, request.rounds)
: compare(request.plain, request.hash);
}
export interface HashPool {
@ -26,7 +43,9 @@ export interface HashPool {
/**
* Runs bcrypt on a small pool of worker threads. bcryptjs is pure JavaScript, so hashing on the
* main thread (sync, or its chunked async API) stalls every other request for the whole hash.
* Workers start on demand and hold the process open only while they have work.
* Workers start on demand and hold the process open only while they have work. A job whose
* worker dies is redone on the main thread, and a worker that dies before answering anything
* moves every later job there too, with one warning.
*/
export function createHashPool(
workerPath: string,
@ -34,22 +53,26 @@ export function createHashPool(
): HashPool {
const workers: HashWorker[] = [];
let nextId = 0;
let unavailable = false;
let warned = false;
const spawn = (): HashWorker => {
const worker = new Worker(workerPath, { execArgv: [] });
worker.unref();
const entry: HashWorker = { worker, pending: new Map() };
const entry: HashWorker = { worker, pending: new Map(), answered: false };
const fail = (): void => {
const index = workers.indexOf(entry);
if (index !== -1) workers.splice(index, 1);
if (!entry.answered) unavailable = true;
for (const { reject } of entry.pending.values()) {
reject(new Error('Password hashing worker stopped'));
reject(new HashWorkerStoppedError());
}
entry.pending.clear();
};
worker.on('message', (reply: HashReply) => {
const pending = entry.pending.get(reply.id);
if (!pending) return;
entry.answered = true;
entry.pending.delete(reply.id);
if (entry.pending.size === 0) worker.unref();
if (reply.ok) pending.resolve(reply.value);
@ -70,15 +93,39 @@ export function createHashPool(
return least;
};
const run = (request: HashRequest): Promise<string | boolean> =>
const runOnWorker = (request: HashRequest): Promise<string | boolean> =>
new Promise((resolve, reject) => {
const entry = pick();
let entry: HashWorker;
try {
entry = pick();
} catch {
unavailable = true;
reject(new HashWorkerStoppedError());
return;
}
const id = nextId++;
entry.pending.set(id, { resolve, reject });
if (entry.pending.size === 1) entry.worker.ref();
entry.worker.postMessage({ id, ...request } satisfies HashJob);
});
const run = async (request: HashRequest): Promise<string | boolean> => {
if (!unavailable) {
try {
return await runOnWorker(request);
} catch (error) {
if (!(error instanceof HashWorkerStoppedError)) throw error;
}
}
if (!warned) {
warned = true;
logger.warn(
'[passwordHashing] A hashing worker stopped; hashing on the main thread, which slows other requests while it runs',
);
}
return runInProcess(request);
};
return {
hash: async (plain, rounds) => (await run({ op: 'hash', plain, rounds })) as string,
compare: async (plain, hash) => (await run({ op: 'compare', plain, hash })) as boolean,

View file

@ -1,3 +1,4 @@
import { logger } from '@librechat/data-schemas';
import { ContentTypes } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
import type { SpeechMessageCache } from './speech';
@ -143,7 +144,44 @@ describe('capSpeechInput', () => {
},
);
it('reads text over the configured cap only up to the cap', () => {
expect(capSpeechInput('a'.repeat(11), { maxInputChars: 10 })).toBe('a'.repeat(10));
it('reads text over the configured cap only up to the cap, saying so without the text', () => {
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
try {
expect(capSpeechInput('a'.repeat(11), { maxInputChars: 10 })).toBe('a'.repeat(10));
expect(warn).toHaveBeenCalledWith(
'[TTS] Text of 11 characters exceeds speech.tts.maxInputChars (10); reading the first 10',
);
} finally {
warn.mockRestore();
}
});
it('stays quiet for text within the cap', () => {
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
try {
capSpeechInput('a'.repeat(10), { maxInputChars: 10 });
expect(warn).not.toHaveBeenCalled();
} finally {
warn.mockRestore();
}
});
});
describe('createSpeechChunkProcessor at the cap', () => {
it('warns once when a message is longer than the cap', async () => {
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const { cache, processChunks } = setup(null, 10);
cache.get.mockResolvedValue({ text: 'b'.repeat(25), complete: false });
try {
await processChunks();
await processChunks();
expect(warn).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
'[TTS] Text of 25 characters exceeds speech.tts.maxInputChars (10); reading the first 10',
);
} finally {
warn.mockRestore();
}
});
});

View file

@ -1,4 +1,4 @@
import { scopedCacheKey } from '@librechat/data-schemas';
import { logger, scopedCacheKey } from '@librechat/data-schemas';
import { Time, getSpeechText, findLastSeparatorIndex } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
@ -17,6 +17,13 @@ export interface SpeechMessageCache {
/** Default cap on the text one text-to-speech request may synthesize. */
export const DEFAULT_TTS_MAX_INPUT_CHARS = 20_000;
/** Lengths only: the text itself may be private. */
function warnSpeechCapped(length: number, maxInputChars: number): void {
logger.warn(
`[TTS] Text of ${length} characters exceeds speech.tts.maxInputChars (${maxInputChars}); reading the first ${maxInputChars}`,
);
}
/**
* Polls a message for the next text to stream as speech. Each call returns the chunks
* finalized since the previous one, or an error string once the message stays missing or
@ -44,6 +51,7 @@ export function createSpeechChunkProcessor({
let notFoundCount = 0;
let noChangeCount = 0;
let processedText = '';
let warnedCap = false;
const cacheKey = scopedCacheKey(messageId);
return async function processChunks() {
@ -68,6 +76,10 @@ export function createSpeechChunkProcessor({
/** A message longer than the cap is read up to the cap, as if it ended there. */
const capped = fullText.length >= maxInputChars;
const text = capped ? fullText.slice(0, maxInputChars) : fullText;
if (fullText.length > maxInputChars && !warnedCap) {
warnedCap = true;
warnSpeechCapped(fullText.length, maxInputChars);
}
const complete =
capped ||
(typeof message !== 'string' && (!('complete' in message) || message.complete !== false));
@ -109,5 +121,9 @@ export function capSpeechInput(input: unknown, ttsConfig?: { maxInputChars?: num
return '';
}
const maxInputChars = ttsConfig?.maxInputChars ?? DEFAULT_TTS_MAX_INPUT_CHARS;
return input.length > maxInputChars ? input.slice(0, maxInputChars) : input;
if (input.length <= maxInputChars) {
return input;
}
warnSpeechCapped(input.length, maxInputChars);
return input.slice(0, maxInputChars);
}

View file

@ -1,5 +1,7 @@
/** `bcryptjs` ships no types. Declared narrowly: only the synchronous calls run, inside the hash worker. */
/** `bcryptjs` ships no types. Declared narrowly: the sync calls run in the hash worker, the async ones only when it cannot. */
declare module 'bcryptjs' {
export function hashSync(plain: string, rounds: number): string;
export function compareSync(plain: string, hash: string): boolean;
export function hash(plain: string, rounds: number): Promise<string>;
export function compare(plain: string, hash: string): Promise<boolean>;
}

View file

@ -2,6 +2,7 @@ import axios from 'axios';
import { Buffer } from 'buffer';
import { logger } from '@librechat/data-schemas';
import type { AxiosInstance, AxiosProxyConfig, AxiosError } from 'axios';
import { isClientDisconnect } from './streams';
/**
* Render `error.response.data` as a small, log-safe value. Axios surfaces
@ -46,6 +47,11 @@ export const logAxiosError = ({
message: string;
error: AxiosError | Error | unknown;
}): string => {
if (isClientDisconnect(error)) {
const logMessage = `${message} Client disconnected`;
logger.debug(logMessage);
return logMessage;
}
let logMessage = message;
try {
const stack =

View file

@ -1,5 +1,8 @@
import { logger } from '@librechat/data-schemas';
import { PassThrough, Readable } from 'node:stream';
import { pipeSegment, pipeToResponse } from './streams';
import { pipeSegment, pipeToResponse, isClientDisconnect } from './streams';
import { isAbortError } from './errors';
import { logAxiosError } from './axios';
function failingSource(): Readable {
return new Readable({
@ -23,15 +26,49 @@ describe('pipeToResponse', () => {
expect(res.destroyed).toBe(true);
});
it('destroys the upstream when the client disconnects', async () => {
it('destroys the upstream without reporting a failure when the client disconnects', async () => {
const source = new PassThrough();
const res = new PassThrough();
const onError = jest.fn();
pipeToResponse(source, res);
pipeToResponse(source, res, onError);
res.destroy();
await new Promise((resolve) => source.on('close', resolve));
await new Promise((resolve) => setImmediate(resolve));
expect(source.destroyed).toBe(true);
expect(onError).not.toHaveBeenCalled();
});
});
describe('pipeToResponse after the upstream ended', () => {
it('treats a response that closed before piping began as a cancellation', async () => {
const source = new PassThrough();
const res = new PassThrough();
const onError = jest.fn();
res.destroy();
await new Promise((resolve) => setImmediate(resolve));
pipeToResponse(source, res, onError);
await new Promise((resolve) => source.on('close', resolve));
await new Promise((resolve) => setImmediate(resolve));
expect(source.destroyed).toBe(true);
expect(onError).not.toHaveBeenCalled();
});
it('treats a client leaving while the response flushes as a cancellation', async () => {
const source = Readable.from([Buffer.from('audio')]);
const res = new PassThrough({ highWaterMark: 1 });
const onError = jest.fn();
pipeToResponse(source, res, onError);
await new Promise((resolve) => source.on('close', resolve));
res.destroy();
await new Promise((resolve) => setImmediate(resolve));
expect(source.destroyed).toBe(true);
expect(onError).not.toHaveBeenCalled();
});
});
@ -59,10 +96,31 @@ describe('pipeSegment', () => {
res.destroy();
await expect(pending).rejects.toThrow('Client disconnected');
const error = await pending.catch((reason: unknown) => reason);
expect(isClientDisconnect(error)).toBe(true);
expect(isAbortError(error)).toBe(true);
expect(source.destroyed).toBe(true);
});
it('logs a client disconnect at debug level, not as a failure', async () => {
const debug = jest.spyOn(logger, 'debug').mockImplementation(() => logger);
const error = jest.spyOn(logger, 'error').mockImplementation(() => logger);
const res = new PassThrough();
res.destroy();
try {
const reason = await pipeSegment(new PassThrough(), res, { end: false }).catch(
(rejected: unknown) => rejected,
);
logAxiosError({ message: '[TTS] segment:', error: reason });
expect(debug).toHaveBeenCalledWith('[TTS] segment: Client disconnected');
expect(error).not.toHaveBeenCalled();
} finally {
debug.mockRestore();
error.mockRestore();
}
});
it('rejects at once when the client left before the segment started', async () => {
const source = new PassThrough();
const res = new PassThrough();

View file

@ -1,17 +1,49 @@
import { pipeline } from 'node:stream';
import type { Readable, Writable } from 'node:stream';
/** The client went away mid-response: a cancellation, not an upstream failure. */
export class ClientDisconnectedError extends Error {
name = 'AbortError';
constructor() {
super('Client disconnected');
}
}
export function isClientDisconnect(error: unknown): error is ClientDisconnectedError {
return error instanceof ClientDisconnectedError;
}
/**
* Streams `source` into `res` and ends it. An upstream failure tears the response down instead
* of surfacing as an uncaught stream error, and a client disconnect destroys the upstream.
* of surfacing as an uncaught stream error, and a client disconnect destroys the upstream
* without reaching `onError`, since nothing failed.
*/
export function pipeToResponse(
source: Readable,
res: Writable,
onError?: (error: Error) => void,
): void {
/** A client can leave while the upstream is still being fetched; `pipeline` would throw. */
if (res.destroyed) {
source.destroy();
return;
}
let clientLeft = false;
let sourceEnded = false;
source.once('end', () => {
sourceEnded = true;
});
/**
* Registered before `pipeline`, so it runs before the upstream is torn down in response. A
* source that already ended may be destroyed while the response flushes; only one that broke
* off makes the close an upstream failure.
*/
res.once('close', () => {
clientLeft = !res.writableFinished && (sourceEnded || !source.destroyed);
});
pipeline(source, res, (error) => {
if (error) {
if (error && !clientLeft) {
onError?.(error);
}
});
@ -46,7 +78,7 @@ export function pipeSegment(
/** A client that left while the previous segment was being fetched already emitted close. */
if (res.destroyed || res.writableEnded) {
source.destroy();
reject(new Error('Client disconnected'));
reject(new ClientDisconnectedError());
return;
}
const cleanup = (): void => {
@ -72,7 +104,7 @@ export function pipeSegment(
const onClose = (): void => {
cleanup();
source.destroy();
reject(new Error('Client disconnected'));
reject(new ClientDisconnectedError());
};
source.on('end', onEnd);
source.on('error', onError);