mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 21:08:17 +00:00
🔕 fix: Quiet Client Disconnects, Fall Back When Hash Workers Fail, Log Speech Truncation (#17278)
This commit is contained in:
parent
6c0d3a7cbb
commit
ded1e734e0
8 changed files with 243 additions and 21 deletions
|
|
@ -1,5 +1,6 @@
|
|||
import path from 'node:path';
|
||||
import { hashSync } from 'bcryptjs';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { createHashPool } from './hashing';
|
||||
|
||||
const workerPath = path.join(
|
||||
|
|
@ -39,13 +40,35 @@ describe('bcrypt hash pool', () => {
|
|||
expect(worstLagMs).toBeLessThan(50);
|
||||
});
|
||||
|
||||
it('rejects a job whose worker dies instead of leaving it pending', async () => {
|
||||
it('finishes a job on the main thread when its worker dies, warning once', async () => {
|
||||
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
|
||||
const crashing = createHashPool(
|
||||
path.join(__dirname, '__fixtures__', 'crashing-hash-worker.cjs'),
|
||||
1,
|
||||
);
|
||||
try {
|
||||
const hash = await crashing.hash('secret', 4);
|
||||
|
||||
await expect(crashing.hash('secret', 4)).rejects.toThrow('Password hashing worker stopped');
|
||||
await crashing.close();
|
||||
await expect(crashing.compare('secret', hash)).resolves.toBe(true);
|
||||
await expect(crashing.compare('other', hash)).resolves.toBe(false);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
await crashing.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('hashes on the main thread when the worker cannot start', async () => {
|
||||
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
|
||||
const missing = createHashPool(path.join(__dirname, '__fixtures__', 'missing-worker.cjs'), 1);
|
||||
try {
|
||||
const hash = await missing.hash('secret', 4);
|
||||
|
||||
await expect(missing.compare('secret', hash)).resolves.toBe(true);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
await missing.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import path from 'node:path';
|
||||
import { compare, hash } from 'bcryptjs';
|
||||
import { Worker } from 'node:worker_threads';
|
||||
import { availableParallelism } from 'node:os';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type { HashJob, HashReply } from './hash-worker';
|
||||
|
||||
type HashRequest =
|
||||
|
|
@ -15,6 +17,21 @@ interface Pending {
|
|||
interface HashWorker {
|
||||
worker: Worker;
|
||||
pending: Map<number, Pending>;
|
||||
/** Whether this worker ever answered; one that dies first means workers cannot start here. */
|
||||
answered: boolean;
|
||||
}
|
||||
|
||||
class HashWorkerStoppedError extends Error {
|
||||
constructor() {
|
||||
super('Password hashing worker stopped');
|
||||
}
|
||||
}
|
||||
|
||||
/** The main-thread path: slower under load, but sign-in keeps working without workers. */
|
||||
function runInProcess(request: HashRequest): Promise<string | boolean> {
|
||||
return request.op === 'hash'
|
||||
? hash(request.plain, request.rounds)
|
||||
: compare(request.plain, request.hash);
|
||||
}
|
||||
|
||||
export interface HashPool {
|
||||
|
|
@ -26,7 +43,9 @@ export interface HashPool {
|
|||
/**
|
||||
* Runs bcrypt on a small pool of worker threads. bcryptjs is pure JavaScript, so hashing on the
|
||||
* main thread (sync, or its chunked async API) stalls every other request for the whole hash.
|
||||
* Workers start on demand and hold the process open only while they have work.
|
||||
* Workers start on demand and hold the process open only while they have work. A job whose
|
||||
* worker dies is redone on the main thread, and a worker that dies before answering anything
|
||||
* moves every later job there too, with one warning.
|
||||
*/
|
||||
export function createHashPool(
|
||||
workerPath: string,
|
||||
|
|
@ -34,22 +53,26 @@ export function createHashPool(
|
|||
): HashPool {
|
||||
const workers: HashWorker[] = [];
|
||||
let nextId = 0;
|
||||
let unavailable = false;
|
||||
let warned = false;
|
||||
|
||||
const spawn = (): HashWorker => {
|
||||
const worker = new Worker(workerPath, { execArgv: [] });
|
||||
worker.unref();
|
||||
const entry: HashWorker = { worker, pending: new Map() };
|
||||
const entry: HashWorker = { worker, pending: new Map(), answered: false };
|
||||
const fail = (): void => {
|
||||
const index = workers.indexOf(entry);
|
||||
if (index !== -1) workers.splice(index, 1);
|
||||
if (!entry.answered) unavailable = true;
|
||||
for (const { reject } of entry.pending.values()) {
|
||||
reject(new Error('Password hashing worker stopped'));
|
||||
reject(new HashWorkerStoppedError());
|
||||
}
|
||||
entry.pending.clear();
|
||||
};
|
||||
worker.on('message', (reply: HashReply) => {
|
||||
const pending = entry.pending.get(reply.id);
|
||||
if (!pending) return;
|
||||
entry.answered = true;
|
||||
entry.pending.delete(reply.id);
|
||||
if (entry.pending.size === 0) worker.unref();
|
||||
if (reply.ok) pending.resolve(reply.value);
|
||||
|
|
@ -70,15 +93,39 @@ export function createHashPool(
|
|||
return least;
|
||||
};
|
||||
|
||||
const run = (request: HashRequest): Promise<string | boolean> =>
|
||||
const runOnWorker = (request: HashRequest): Promise<string | boolean> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const entry = pick();
|
||||
let entry: HashWorker;
|
||||
try {
|
||||
entry = pick();
|
||||
} catch {
|
||||
unavailable = true;
|
||||
reject(new HashWorkerStoppedError());
|
||||
return;
|
||||
}
|
||||
const id = nextId++;
|
||||
entry.pending.set(id, { resolve, reject });
|
||||
if (entry.pending.size === 1) entry.worker.ref();
|
||||
entry.worker.postMessage({ id, ...request } satisfies HashJob);
|
||||
});
|
||||
|
||||
const run = async (request: HashRequest): Promise<string | boolean> => {
|
||||
if (!unavailable) {
|
||||
try {
|
||||
return await runOnWorker(request);
|
||||
} catch (error) {
|
||||
if (!(error instanceof HashWorkerStoppedError)) throw error;
|
||||
}
|
||||
}
|
||||
if (!warned) {
|
||||
warned = true;
|
||||
logger.warn(
|
||||
'[passwordHashing] A hashing worker stopped; hashing on the main thread, which slows other requests while it runs',
|
||||
);
|
||||
}
|
||||
return runInProcess(request);
|
||||
};
|
||||
|
||||
return {
|
||||
hash: async (plain, rounds) => (await run({ op: 'hash', plain, rounds })) as string,
|
||||
compare: async (plain, hash) => (await run({ op: 'compare', plain, hash })) as boolean,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import { ContentTypes } from 'librechat-data-provider';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
import type { SpeechMessageCache } from './speech';
|
||||
|
|
@ -143,7 +144,44 @@ describe('capSpeechInput', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it('reads text over the configured cap only up to the cap', () => {
|
||||
expect(capSpeechInput('a'.repeat(11), { maxInputChars: 10 })).toBe('a'.repeat(10));
|
||||
it('reads text over the configured cap only up to the cap, saying so without the text', () => {
|
||||
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
|
||||
try {
|
||||
expect(capSpeechInput('a'.repeat(11), { maxInputChars: 10 })).toBe('a'.repeat(10));
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[TTS] Text of 11 characters exceeds speech.tts.maxInputChars (10); reading the first 10',
|
||||
);
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('stays quiet for text within the cap', () => {
|
||||
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
|
||||
try {
|
||||
capSpeechInput('a'.repeat(10), { maxInputChars: 10 });
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('createSpeechChunkProcessor at the cap', () => {
|
||||
it('warns once when a message is longer than the cap', async () => {
|
||||
const warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
|
||||
const { cache, processChunks } = setup(null, 10);
|
||||
cache.get.mockResolvedValue({ text: 'b'.repeat(25), complete: false });
|
||||
try {
|
||||
await processChunks();
|
||||
await processChunks();
|
||||
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[TTS] Text of 25 characters exceeds speech.tts.maxInputChars (10); reading the first 10',
|
||||
);
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { scopedCacheKey } from '@librechat/data-schemas';
|
||||
import { logger, scopedCacheKey } from '@librechat/data-schemas';
|
||||
import { Time, getSpeechText, findLastSeparatorIndex } from 'librechat-data-provider';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
|
||||
|
|
@ -17,6 +17,13 @@ export interface SpeechMessageCache {
|
|||
/** Default cap on the text one text-to-speech request may synthesize. */
|
||||
export const DEFAULT_TTS_MAX_INPUT_CHARS = 20_000;
|
||||
|
||||
/** Lengths only: the text itself may be private. */
|
||||
function warnSpeechCapped(length: number, maxInputChars: number): void {
|
||||
logger.warn(
|
||||
`[TTS] Text of ${length} characters exceeds speech.tts.maxInputChars (${maxInputChars}); reading the first ${maxInputChars}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls a message for the next text to stream as speech. Each call returns the chunks
|
||||
* finalized since the previous one, or an error string once the message stays missing or
|
||||
|
|
@ -44,6 +51,7 @@ export function createSpeechChunkProcessor({
|
|||
let notFoundCount = 0;
|
||||
let noChangeCount = 0;
|
||||
let processedText = '';
|
||||
let warnedCap = false;
|
||||
const cacheKey = scopedCacheKey(messageId);
|
||||
|
||||
return async function processChunks() {
|
||||
|
|
@ -68,6 +76,10 @@ export function createSpeechChunkProcessor({
|
|||
/** A message longer than the cap is read up to the cap, as if it ended there. */
|
||||
const capped = fullText.length >= maxInputChars;
|
||||
const text = capped ? fullText.slice(0, maxInputChars) : fullText;
|
||||
if (fullText.length > maxInputChars && !warnedCap) {
|
||||
warnedCap = true;
|
||||
warnSpeechCapped(fullText.length, maxInputChars);
|
||||
}
|
||||
const complete =
|
||||
capped ||
|
||||
(typeof message !== 'string' && (!('complete' in message) || message.complete !== false));
|
||||
|
|
@ -109,5 +121,9 @@ export function capSpeechInput(input: unknown, ttsConfig?: { maxInputChars?: num
|
|||
return '';
|
||||
}
|
||||
const maxInputChars = ttsConfig?.maxInputChars ?? DEFAULT_TTS_MAX_INPUT_CHARS;
|
||||
return input.length > maxInputChars ? input.slice(0, maxInputChars) : input;
|
||||
if (input.length <= maxInputChars) {
|
||||
return input;
|
||||
}
|
||||
warnSpeechCapped(input.length, maxInputChars);
|
||||
return input.slice(0, maxInputChars);
|
||||
}
|
||||
|
|
|
|||
4
packages/api/src/types/bcryptjs.d.ts
vendored
4
packages/api/src/types/bcryptjs.d.ts
vendored
|
|
@ -1,5 +1,7 @@
|
|||
/** `bcryptjs` ships no types. Declared narrowly: only the synchronous calls run, inside the hash worker. */
|
||||
/** `bcryptjs` ships no types. Declared narrowly: the sync calls run in the hash worker, the async ones only when it cannot. */
|
||||
declare module 'bcryptjs' {
|
||||
export function hashSync(plain: string, rounds: number): string;
|
||||
export function compareSync(plain: string, hash: string): boolean;
|
||||
export function hash(plain: string, rounds: number): Promise<string>;
|
||||
export function compare(plain: string, hash: string): Promise<boolean>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import axios from 'axios';
|
|||
import { Buffer } from 'buffer';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type { AxiosInstance, AxiosProxyConfig, AxiosError } from 'axios';
|
||||
import { isClientDisconnect } from './streams';
|
||||
|
||||
/**
|
||||
* Render `error.response.data` as a small, log-safe value. Axios surfaces
|
||||
|
|
@ -46,6 +47,11 @@ export const logAxiosError = ({
|
|||
message: string;
|
||||
error: AxiosError | Error | unknown;
|
||||
}): string => {
|
||||
if (isClientDisconnect(error)) {
|
||||
const logMessage = `${message} Client disconnected`;
|
||||
logger.debug(logMessage);
|
||||
return logMessage;
|
||||
}
|
||||
let logMessage = message;
|
||||
try {
|
||||
const stack =
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import { PassThrough, Readable } from 'node:stream';
|
||||
import { pipeSegment, pipeToResponse } from './streams';
|
||||
import { pipeSegment, pipeToResponse, isClientDisconnect } from './streams';
|
||||
import { isAbortError } from './errors';
|
||||
import { logAxiosError } from './axios';
|
||||
|
||||
function failingSource(): Readable {
|
||||
return new Readable({
|
||||
|
|
@ -23,15 +26,49 @@ describe('pipeToResponse', () => {
|
|||
expect(res.destroyed).toBe(true);
|
||||
});
|
||||
|
||||
it('destroys the upstream when the client disconnects', async () => {
|
||||
it('destroys the upstream without reporting a failure when the client disconnects', async () => {
|
||||
const source = new PassThrough();
|
||||
const res = new PassThrough();
|
||||
const onError = jest.fn();
|
||||
|
||||
pipeToResponse(source, res);
|
||||
pipeToResponse(source, res, onError);
|
||||
res.destroy();
|
||||
await new Promise((resolve) => source.on('close', resolve));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(source.destroyed).toBe(true);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pipeToResponse after the upstream ended', () => {
|
||||
it('treats a response that closed before piping began as a cancellation', async () => {
|
||||
const source = new PassThrough();
|
||||
const res = new PassThrough();
|
||||
const onError = jest.fn();
|
||||
res.destroy();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
pipeToResponse(source, res, onError);
|
||||
await new Promise((resolve) => source.on('close', resolve));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(source.destroyed).toBe(true);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('treats a client leaving while the response flushes as a cancellation', async () => {
|
||||
const source = Readable.from([Buffer.from('audio')]);
|
||||
const res = new PassThrough({ highWaterMark: 1 });
|
||||
const onError = jest.fn();
|
||||
|
||||
pipeToResponse(source, res, onError);
|
||||
await new Promise((resolve) => source.on('close', resolve));
|
||||
res.destroy();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(source.destroyed).toBe(true);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -59,10 +96,31 @@ describe('pipeSegment', () => {
|
|||
|
||||
res.destroy();
|
||||
|
||||
await expect(pending).rejects.toThrow('Client disconnected');
|
||||
const error = await pending.catch((reason: unknown) => reason);
|
||||
expect(isClientDisconnect(error)).toBe(true);
|
||||
expect(isAbortError(error)).toBe(true);
|
||||
expect(source.destroyed).toBe(true);
|
||||
});
|
||||
|
||||
it('logs a client disconnect at debug level, not as a failure', async () => {
|
||||
const debug = jest.spyOn(logger, 'debug').mockImplementation(() => logger);
|
||||
const error = jest.spyOn(logger, 'error').mockImplementation(() => logger);
|
||||
const res = new PassThrough();
|
||||
res.destroy();
|
||||
try {
|
||||
const reason = await pipeSegment(new PassThrough(), res, { end: false }).catch(
|
||||
(rejected: unknown) => rejected,
|
||||
);
|
||||
logAxiosError({ message: '[TTS] segment:', error: reason });
|
||||
|
||||
expect(debug).toHaveBeenCalledWith('[TTS] segment: Client disconnected');
|
||||
expect(error).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
debug.mockRestore();
|
||||
error.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects at once when the client left before the segment started', async () => {
|
||||
const source = new PassThrough();
|
||||
const res = new PassThrough();
|
||||
|
|
|
|||
|
|
@ -1,17 +1,49 @@
|
|||
import { pipeline } from 'node:stream';
|
||||
import type { Readable, Writable } from 'node:stream';
|
||||
|
||||
/** The client went away mid-response: a cancellation, not an upstream failure. */
|
||||
export class ClientDisconnectedError extends Error {
|
||||
name = 'AbortError';
|
||||
|
||||
constructor() {
|
||||
super('Client disconnected');
|
||||
}
|
||||
}
|
||||
|
||||
export function isClientDisconnect(error: unknown): error is ClientDisconnectedError {
|
||||
return error instanceof ClientDisconnectedError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams `source` into `res` and ends it. An upstream failure tears the response down instead
|
||||
* of surfacing as an uncaught stream error, and a client disconnect destroys the upstream.
|
||||
* of surfacing as an uncaught stream error, and a client disconnect destroys the upstream
|
||||
* without reaching `onError`, since nothing failed.
|
||||
*/
|
||||
export function pipeToResponse(
|
||||
source: Readable,
|
||||
res: Writable,
|
||||
onError?: (error: Error) => void,
|
||||
): void {
|
||||
/** A client can leave while the upstream is still being fetched; `pipeline` would throw. */
|
||||
if (res.destroyed) {
|
||||
source.destroy();
|
||||
return;
|
||||
}
|
||||
let clientLeft = false;
|
||||
let sourceEnded = false;
|
||||
source.once('end', () => {
|
||||
sourceEnded = true;
|
||||
});
|
||||
/**
|
||||
* Registered before `pipeline`, so it runs before the upstream is torn down in response. A
|
||||
* source that already ended may be destroyed while the response flushes; only one that broke
|
||||
* off makes the close an upstream failure.
|
||||
*/
|
||||
res.once('close', () => {
|
||||
clientLeft = !res.writableFinished && (sourceEnded || !source.destroyed);
|
||||
});
|
||||
pipeline(source, res, (error) => {
|
||||
if (error) {
|
||||
if (error && !clientLeft) {
|
||||
onError?.(error);
|
||||
}
|
||||
});
|
||||
|
|
@ -46,7 +78,7 @@ export function pipeSegment(
|
|||
/** A client that left while the previous segment was being fetched already emitted close. */
|
||||
if (res.destroyed || res.writableEnded) {
|
||||
source.destroy();
|
||||
reject(new Error('Client disconnected'));
|
||||
reject(new ClientDisconnectedError());
|
||||
return;
|
||||
}
|
||||
const cleanup = (): void => {
|
||||
|
|
@ -72,7 +104,7 @@ export function pipeSegment(
|
|||
const onClose = (): void => {
|
||||
cleanup();
|
||||
source.destroy();
|
||||
reject(new Error('Client disconnected'));
|
||||
reject(new ClientDisconnectedError());
|
||||
};
|
||||
source.on('end', onEnd);
|
||||
source.on('error', onError);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue