mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
🩻 fix: Preserve Workspace Tool HTTP Diagnostics (#15723)
* fix: preserve workspace tool HTTP diagnostics * fix: filter raw workspace diagnostics before logging * fix: preserve workspace cancellation and exact body limits
This commit is contained in:
parent
f50c40e2f5
commit
1b51ec49ee
4 changed files with 305 additions and 62 deletions
|
|
@ -1459,58 +1459,92 @@ describe('createToolExecuteHandler', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('filters thrown foreground errors before result delivery or logging', async () => {
|
||||
const protectedValue = 'PROTECTED-FOREGROUND-ERROR';
|
||||
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
|
||||
loadedTools: [
|
||||
{
|
||||
name: 'throwing_tool',
|
||||
invoke: jest.fn(async () => {
|
||||
throw new Error(protectedValue);
|
||||
}),
|
||||
},
|
||||
] as never[],
|
||||
}));
|
||||
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
try {
|
||||
const handler = createToolExecuteHandler({ loadTools });
|
||||
const [result] = await invokeHandlerWithConfig(
|
||||
handler,
|
||||
[{ id: 'call_filtered_throw', name: 'throwing_tool', args: {} }],
|
||||
{
|
||||
req: {
|
||||
config: {
|
||||
filters: {
|
||||
toolArguments: {
|
||||
pii: {
|
||||
fields: ['output'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{
|
||||
id: 'protected-output',
|
||||
label: 'protected output',
|
||||
regex: 'PROTECTED-[A-Z-]+',
|
||||
},
|
||||
],
|
||||
it.each(['generic', 'workspace', 'workspace-expanded'])(
|
||||
'filters %s foreground errors before result delivery or logging',
|
||||
async (kind) => {
|
||||
const protectedValue = 'PROTECTED-FOREGROUND-ERROR';
|
||||
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
|
||||
loadedTools: [
|
||||
{
|
||||
name: 'throwing_tool',
|
||||
invoke: jest.fn(async () => {
|
||||
if (kind.startsWith('workspace')) {
|
||||
const body =
|
||||
kind === 'workspace-expanded'
|
||||
? '\u0001'.repeat(2000) + protectedValue + '\u0001'.repeat(2000)
|
||||
: protectedValue;
|
||||
throw new WorkspaceToolHttpError('rejected', 503, body);
|
||||
}
|
||||
throw new Error(protectedValue);
|
||||
}),
|
||||
},
|
||||
] as never[],
|
||||
}));
|
||||
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
try {
|
||||
const handler = createToolExecuteHandler({ loadTools });
|
||||
const [result] = await invokeHandlerWithConfig(
|
||||
handler,
|
||||
[{ id: 'call_filtered_throw', name: 'throwing_tool', args: {} }],
|
||||
{
|
||||
req: {
|
||||
config: {
|
||||
filters: {
|
||||
toolArguments: {
|
||||
pii: {
|
||||
fields: ['output'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{
|
||||
id: 'protected-output',
|
||||
label: 'protected output',
|
||||
regex: 'PROTECTED-[A-Z-]+',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
);
|
||||
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.errorMessage).toContain('content_filter_block');
|
||||
expect(result.errorMessage).not.toContain(protectedValue);
|
||||
expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(protectedValue);
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
'[ON_TOOL_EXECUTE] Tool throwing_tool error',
|
||||
expect.objectContaining({ contentFiltered: true }),
|
||||
);
|
||||
} finally {
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.errorMessage).toContain('content_filter_block');
|
||||
expect(result.errorMessage).not.toContain(protectedValue);
|
||||
expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(protectedValue);
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
'[ON_TOOL_EXECUTE] Tool throwing_tool error',
|
||||
expect.objectContaining({ contentFiltered: true }),
|
||||
);
|
||||
} finally {
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('surfaces workspace transport diagnostics thrown by a loaded tool', async () => {
|
||||
const body = '{"code":"ASSIGNMENT_EXPIRED"}';
|
||||
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
|
||||
loadedTools: [
|
||||
{
|
||||
name: 'workspace_command',
|
||||
invoke: jest.fn(async () => {
|
||||
throw new WorkspaceToolHttpError('rejected', 504, body);
|
||||
}),
|
||||
},
|
||||
] as never[],
|
||||
}));
|
||||
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
const [result] = await invokeHandler(createToolExecuteHandler({ loadTools }), [
|
||||
{ id: 'call_command_error', name: 'workspace_command', args: {} },
|
||||
]);
|
||||
expect(result.errorMessage).toContain('upstreamStatus: 504');
|
||||
expect(result.errorMessage).toContain('ASSIGNMENT_EXPIRED');
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
'[ON_TOOL_EXECUTE] Tool workspace_command error',
|
||||
expect.objectContaining({ upstreamStatus: 504, upstreamBody: body }),
|
||||
);
|
||||
});
|
||||
|
||||
it('truncates oversized tool errors in the result and log context', async () => {
|
||||
|
|
@ -5524,6 +5558,42 @@ describe('createToolExecuteHandler', () => {
|
|||
expect(searchWorkspace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([400, 409, 503, 504])(
|
||||
'surfaces workspace HTTP %i in logs and model results',
|
||||
async (status) => {
|
||||
const body = '{"code":"WORKER_BUSY","error":"Worker is busy"}';
|
||||
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
|
||||
const handler = makeReadFileHandler({
|
||||
codeEnvAvailable: true,
|
||||
codeExecutionContext: {
|
||||
baseUrl: 'https://code.example.com',
|
||||
codeSessionKey: 'attached',
|
||||
executionProfile: 'stateful',
|
||||
statefulSessions: true,
|
||||
environmentType: 'attached',
|
||||
},
|
||||
listWorkspaceFiles: jest.fn(async () => {
|
||||
throw new WorkspaceToolHttpError('rejected', status, body);
|
||||
}),
|
||||
});
|
||||
const [result] = await invokeHandler(handler, [
|
||||
{ id: 'call_workspace_error', name: 'list_workspace_files', args: {} },
|
||||
]);
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.errorMessage).toContain(`upstreamStatus: ${status}`);
|
||||
expect(result.errorMessage).toContain('WORKER_BUSY');
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
'[ON_TOOL_EXECUTE] Tool list_workspace_files error',
|
||||
expect.objectContaining({
|
||||
name: 'WorkspaceToolHttpError',
|
||||
upstreamStatus: status,
|
||||
upstreamBody: body,
|
||||
upstreamBodyTruncated: false,
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('lists files through the selected attached worker and forwards cancellation', async () => {
|
||||
const controller = new AbortController();
|
||||
const listWorkspaceFiles = jest.fn(async () => ({
|
||||
|
|
|
|||
|
|
@ -765,6 +765,13 @@ function getSafeToolError(error: unknown): {
|
|||
message,
|
||||
logContext: {
|
||||
name: error instanceof Error ? error.name : typeof error,
|
||||
...(error instanceof WorkspaceToolHttpError
|
||||
? {
|
||||
upstreamStatus: error.upstreamStatus,
|
||||
upstreamBody: error.upstreamBody,
|
||||
upstreamBodyTruncated: error.upstreamBodyTruncated,
|
||||
}
|
||||
: {}),
|
||||
message,
|
||||
messageLength: rawMessage.length,
|
||||
messageTruncated: message.length !== rawMessage.length,
|
||||
|
|
@ -2378,6 +2385,7 @@ async function handleWorkspaceFileRead(
|
|||
content: numbered,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (signal?.aborted === true && isAbortError(error)) throw error;
|
||||
logger.warn(
|
||||
'[handleWorkspaceFileRead] Attached workspace read failed',
|
||||
|
|
@ -2464,6 +2472,7 @@ async function handleWorkspaceSearchCall(
|
|||
content: truncated ? `${content}${truncationNotice}` : content,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (signal?.aborted === true && isAbortError(error)) throw error;
|
||||
logger.warn(
|
||||
'[handleWorkspaceSearchCall] Attached workspace search failed',
|
||||
|
|
@ -2566,6 +2575,7 @@ async function handleWorkspaceListCall(
|
|||
content: `${content}${truncationNotice}`,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError) throw error;
|
||||
if (signal?.aborted === true && isAbortError(error)) throw error;
|
||||
logger.warn(
|
||||
'[handleWorkspaceListCall] Attached workspace file listing failed',
|
||||
|
|
@ -3725,8 +3735,11 @@ async function handleAttachedWorkspaceCreateFileCall({
|
|||
created: result.created,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 409 && !overwrite) {
|
||||
return errorResult(tc, 'File already exists. Pass overwrite: true to replace.');
|
||||
if (error instanceof WorkspaceToolHttpError) {
|
||||
if (error.upstreamStatus === 409 && !overwrite) {
|
||||
error.message += '. File already exists. Pass overwrite: true to replace.';
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (signal?.aborted === true && isAbortError(error)) throw error;
|
||||
logger.warn('[file_authoring] Attached workspace write failed', getSafeErrorMetadata(error));
|
||||
|
|
@ -3798,10 +3811,8 @@ async function handleAttachedWorkspaceEditFileCall({
|
|||
} catch (error) {
|
||||
if (signal?.aborted === true && isAbortError(error)) throw error;
|
||||
if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 400) {
|
||||
return errorResult(
|
||||
tc,
|
||||
'This attached environment must update its LibreChat Code worker before protected files can be edited.',
|
||||
);
|
||||
error.message +=
|
||||
'. This attached environment must update its LibreChat Code worker before protected files can be edited.';
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
|
@ -3828,11 +3839,11 @@ async function handleAttachedWorkspaceEditFileCall({
|
|||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 409) {
|
||||
return errorResult(
|
||||
tc,
|
||||
`The requested text did not match exactly once in "workspace/${path.filePath}". Re-read the file and retry.`,
|
||||
);
|
||||
if (error instanceof WorkspaceToolHttpError) {
|
||||
if (error.upstreamStatus === 409) {
|
||||
error.message += `; The requested text did not match exactly once in "workspace/${path.filePath}". Re-read the file and retry.`;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (signal?.aborted === true && isAbortError(error)) throw error;
|
||||
logger.warn('[file_authoring] Attached workspace edit failed', getSafeErrorMetadata(error));
|
||||
|
|
@ -5963,6 +5974,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
policyError == null
|
||||
? filteredToolOutputResult(tc, backgroundReq, {
|
||||
errorMessage: errorOutput,
|
||||
upstreamBody:
|
||||
toolError instanceof WorkspaceToolHttpError
|
||||
? toolError.upstreamBody
|
||||
: undefined,
|
||||
})
|
||||
: null;
|
||||
const neutralizedError = filteredError?.errorMessage ?? errorOutput;
|
||||
|
|
@ -6373,6 +6388,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
const { message, logContext } = getSafeToolError(toolError);
|
||||
const filteredError = filteredToolOutputResult(tc, req, {
|
||||
errorMessage: message,
|
||||
upstreamBody:
|
||||
toolError instanceof WorkspaceToolHttpError
|
||||
? toolError.upstreamBody
|
||||
: undefined,
|
||||
});
|
||||
if (filteredError != null) {
|
||||
logger.error(`[ON_TOOL_EXECUTE] Tool ${tc.name} error`, {
|
||||
|
|
@ -6781,6 +6800,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
|
|||
const req = mergedConfigurable?.req as ServerRequest | undefined;
|
||||
const filteredError = filteredToolOutputResult(tc, req, {
|
||||
errorMessage: message,
|
||||
upstreamBody:
|
||||
toolError instanceof WorkspaceToolHttpError
|
||||
? toolError.upstreamBody
|
||||
: undefined,
|
||||
});
|
||||
if (filteredError != null) {
|
||||
logToolFailure({
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ describe('executeWorkspaceTool', () => {
|
|||
).rejects.toMatchObject({ reason: 'invalid' });
|
||||
});
|
||||
|
||||
test('surfaces bounded upstream failures without returning their body', async () => {
|
||||
test('preserves the upstream status when the error body stalls', async () => {
|
||||
const cancel = jest.fn();
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(
|
||||
|
|
@ -296,10 +296,105 @@ describe('executeWorkspaceTool', () => {
|
|||
},
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toMatchObject({ reason: 'rejected', upstreamStatus: 503 });
|
||||
).rejects.toMatchObject({
|
||||
reason: 'rejected',
|
||||
upstreamStatus: 503,
|
||||
upstreamBodyTruncated: true,
|
||||
});
|
||||
expect(cancel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test.each([400, 409, 503, 504])('preserves HTTP %i and its diagnostic body', async (status) => {
|
||||
const body = JSON.stringify({ code: 'ASSIGNMENT_EXPIRED', error: 'Assignment expired' });
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com',
|
||||
authHeaders: {},
|
||||
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
|
||||
fetchImpl: jest.fn(async () => new Response(body, { status })),
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
reason: 'rejected',
|
||||
upstreamStatus: status,
|
||||
upstreamBody: body,
|
||||
upstreamBodyTruncated: false,
|
||||
message: expect.stringContaining(`upstreamStatus: ${status}`),
|
||||
});
|
||||
});
|
||||
|
||||
test.each([4095, 4096, 4097])(
|
||||
'reports truncation correctly for a %i-byte error body',
|
||||
async (size) => {
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com',
|
||||
authHeaders: {},
|
||||
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
|
||||
fetchImpl: jest.fn(async () => new Response('x'.repeat(size), { status: 503 })),
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
upstreamStatus: 503,
|
||||
upstreamBody: 'x'.repeat(Math.min(size, 4096)),
|
||||
upstreamBodyTruncated: size > 4096,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test('preserves caller cancellation during a rejected response body read', async () => {
|
||||
const controller = new AbortController();
|
||||
const cancel = jest.fn();
|
||||
const request = executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com',
|
||||
authHeaders: {},
|
||||
signal: controller.signal,
|
||||
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
|
||||
fetchImpl: jest.fn(async () => new Response(new ReadableStream({ cancel }), { status: 503 })),
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
controller.abort();
|
||||
await expect(request).rejects.toBe(controller.signal.reason);
|
||||
expect(cancel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('bounds a streaming error body and cancels the unread remainder', async () => {
|
||||
const cancel = jest.fn();
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode('x'.repeat(10_000)));
|
||||
},
|
||||
cancel,
|
||||
});
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com',
|
||||
authHeaders: {},
|
||||
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
|
||||
fetchImpl: jest.fn(async () => new Response(body, { status: 504 })),
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
upstreamStatus: 504,
|
||||
upstreamBody: 'x'.repeat(4096),
|
||||
upstreamBodyTruncated: true,
|
||||
});
|
||||
expect(cancel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('retains HTTP status when reading the error body fails', async () => {
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.error(new Error('socket closed'));
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
executeWorkspaceTool({
|
||||
baseURL: 'https://code.example.com',
|
||||
authHeaders: {},
|
||||
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
|
||||
fetchImpl: jest.fn(async () => new Response(body, { status: 503 })),
|
||||
}),
|
||||
).rejects.toMatchObject({ upstreamStatus: 503, upstreamBodyTruncated: true });
|
||||
});
|
||||
|
||||
test('validates bounded search matches before returning them', async () => {
|
||||
const fetchImpl = jest.fn().mockResolvedValue(
|
||||
new Response(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024;
|
|||
const MAX_COMMAND_SIGNAL_LENGTH = 32;
|
||||
const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000;
|
||||
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
|
||||
const MAX_ERROR_BODY_BYTES = 4096;
|
||||
const ERROR_BODY_TIMEOUT_MS = 1000;
|
||||
const READ_RESULT_KEYS = new Set([
|
||||
'protocolVersion',
|
||||
'operation',
|
||||
|
|
@ -246,12 +248,64 @@ export class WorkspaceToolHttpError extends Error {
|
|||
constructor(
|
||||
public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed',
|
||||
public readonly upstreamStatus?: number,
|
||||
public readonly upstreamBody?: string,
|
||||
public readonly upstreamBodyTruncated = false,
|
||||
) {
|
||||
super(`Workspace tool request ${reason}`);
|
||||
super(
|
||||
`Workspace tool request ${reason}` +
|
||||
(upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) +
|
||||
(upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') +
|
||||
(upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''),
|
||||
);
|
||||
this.name = 'WorkspaceToolHttpError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
|
||||
async function readErrorBody(
|
||||
response: Response,
|
||||
signal: AbortSignal,
|
||||
): Promise<{
|
||||
body: string;
|
||||
truncated: boolean;
|
||||
}> {
|
||||
if (!response.body) return { body: '', truncated: false };
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let body = '';
|
||||
let bytes = 0;
|
||||
let complete = false;
|
||||
let interrupted = false;
|
||||
const cancel = () => {
|
||||
interrupted = true;
|
||||
void reader.cancel().catch(() => undefined);
|
||||
};
|
||||
const timer = setTimeout(cancel, ERROR_BODY_TIMEOUT_MS);
|
||||
signal.addEventListener('abort', cancel, { once: true });
|
||||
try {
|
||||
if (signal.aborted) return { body, truncated: true };
|
||||
while (bytes <= MAX_ERROR_BODY_BYTES) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
complete = !interrupted;
|
||||
body += decoder.decode();
|
||||
break;
|
||||
}
|
||||
const remaining = MAX_ERROR_BODY_BYTES - bytes;
|
||||
body += decoder.decode(value.subarray(0, remaining), { stream: true });
|
||||
bytes += value.byteLength;
|
||||
}
|
||||
} catch {
|
||||
complete = false;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener('abort', cancel);
|
||||
cancel();
|
||||
reader.releaseLock();
|
||||
}
|
||||
return { body, truncated: !complete || signal.aborted };
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value != null;
|
||||
}
|
||||
|
|
@ -639,8 +693,9 @@ export async function executeWorkspaceTool({
|
|||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
throw new WorkspaceToolHttpError('rejected', response.status);
|
||||
const { body, truncated } = await readErrorBody(response, requestSignal);
|
||||
signal?.throwIfAborted();
|
||||
throw new WorkspaceToolHttpError('rejected', response.status, body, truncated);
|
||||
}
|
||||
const result = await readBoundedJson(response, requestSignal);
|
||||
if (!isValidResult(request, result)) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue