🩻 fix: Preserve Workspace Tool HTTP Diagnostics (#15723)

* fix: preserve workspace tool HTTP diagnostics

* fix: filter raw workspace diagnostics before logging

* fix: preserve workspace cancellation and exact body limits
This commit is contained in:
Danny Avila 2026-09-08 07:09:06 -04:00 • committed by GitHub
parent f50c40e2f5
commit 1b51ec49ee
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 305 additions and 62 deletions

View file

@ -1459,58 +1459,92 @@ describe('createToolExecuteHandler', () => {
}
});
it('filters thrown foreground errors before result delivery or logging', async () => {
const protectedValue = 'PROTECTED-FOREGROUND-ERROR';
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
loadedTools: [
{
name: 'throwing_tool',
invoke: jest.fn(async () => {
throw new Error(protectedValue);
}),
},
] as never[],
}));
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
try {
const handler = createToolExecuteHandler({ loadTools });
const [result] = await invokeHandlerWithConfig(
handler,
[{ id: 'call_filtered_throw', name: 'throwing_tool', args: {} }],
{
req: {
config: {
filters: {
toolArguments: {
pii: {
fields: ['output'],
starterPatterns: [],
customPatterns: [
{
id: 'protected-output',
label: 'protected output',
regex: 'PROTECTED-[A-Z-]+',
},
],
it.each(['generic', 'workspace', 'workspace-expanded'])(
'filters %s foreground errors before result delivery or logging',
async (kind) => {
const protectedValue = 'PROTECTED-FOREGROUND-ERROR';
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
loadedTools: [
{
name: 'throwing_tool',
invoke: jest.fn(async () => {
if (kind.startsWith('workspace')) {
const body =
kind === 'workspace-expanded'
? '\u0001'.repeat(2000) + protectedValue + '\u0001'.repeat(2000)
: protectedValue;
throw new WorkspaceToolHttpError('rejected', 503, body);
}
throw new Error(protectedValue);
}),
},
] as never[],
}));
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
try {
const handler = createToolExecuteHandler({ loadTools });
const [result] = await invokeHandlerWithConfig(
handler,
[{ id: 'call_filtered_throw', name: 'throwing_tool', args: {} }],
{
req: {
config: {
filters: {
toolArguments: {
pii: {
fields: ['output'],
starterPatterns: [],
customPatterns: [
{
id: 'protected-output',
label: 'protected output',
regex: 'PROTECTED-[A-Z-]+',
},
],
},
},
},
},
},
},
},
);
);
expect(result.status).toBe('error');
expect(result.errorMessage).toContain('content_filter_block');
expect(result.errorMessage).not.toContain(protectedValue);
expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(protectedValue);
expect(errorSpy).toHaveBeenCalledWith(
'[ON_TOOL_EXECUTE] Tool throwing_tool error',
expect.objectContaining({ contentFiltered: true }),
);
} finally {
errorSpy.mockRestore();
}
expect(result.status).toBe('error');
expect(result.errorMessage).toContain('content_filter_block');
expect(result.errorMessage).not.toContain(protectedValue);
expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(protectedValue);
expect(errorSpy).toHaveBeenCalledWith(
'[ON_TOOL_EXECUTE] Tool throwing_tool error',
expect.objectContaining({ contentFiltered: true }),
);
} finally {
errorSpy.mockRestore();
}
},
);
it('surfaces workspace transport diagnostics thrown by a loaded tool', async () => {
const body = '{"code":"ASSIGNMENT_EXPIRED"}';
const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({
loadedTools: [
{
name: 'workspace_command',
invoke: jest.fn(async () => {
throw new WorkspaceToolHttpError('rejected', 504, body);
}),
},
] as never[],
}));
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
const [result] = await invokeHandler(createToolExecuteHandler({ loadTools }), [
{ id: 'call_command_error', name: 'workspace_command', args: {} },
]);
expect(result.errorMessage).toContain('upstreamStatus: 504');
expect(result.errorMessage).toContain('ASSIGNMENT_EXPIRED');
expect(errorSpy).toHaveBeenCalledWith(
'[ON_TOOL_EXECUTE] Tool workspace_command error',
expect.objectContaining({ upstreamStatus: 504, upstreamBody: body }),
);
});
it('truncates oversized tool errors in the result and log context', async () => {
@ -5524,6 +5558,42 @@ describe('createToolExecuteHandler', () => {
expect(searchWorkspace).not.toHaveBeenCalled();
});
it.each([400, 409, 503, 504])(
'surfaces workspace HTTP %i in logs and model results',
async (status) => {
const body = '{"code":"WORKER_BUSY","error":"Worker is busy"}';
const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger);
const handler = makeReadFileHandler({
codeEnvAvailable: true,
codeExecutionContext: {
baseUrl: 'https://code.example.com',
codeSessionKey: 'attached',
executionProfile: 'stateful',
statefulSessions: true,
environmentType: 'attached',
},
listWorkspaceFiles: jest.fn(async () => {
throw new WorkspaceToolHttpError('rejected', status, body);
}),
});
const [result] = await invokeHandler(handler, [
{ id: 'call_workspace_error', name: 'list_workspace_files', args: {} },
]);
expect(result.status).toBe('error');
expect(result.errorMessage).toContain(`upstreamStatus: ${status}`);
expect(result.errorMessage).toContain('WORKER_BUSY');
expect(errorSpy).toHaveBeenCalledWith(
'[ON_TOOL_EXECUTE] Tool list_workspace_files error',
expect.objectContaining({
name: 'WorkspaceToolHttpError',
upstreamStatus: status,
upstreamBody: body,
upstreamBodyTruncated: false,
}),
);
},
);
it('lists files through the selected attached worker and forwards cancellation', async () => {
const controller = new AbortController();
const listWorkspaceFiles = jest.fn(async () => ({

View file

@ -765,6 +765,13 @@ function getSafeToolError(error: unknown): {
message,
logContext: {
name: error instanceof Error ? error.name : typeof error,
...(error instanceof WorkspaceToolHttpError
? {
upstreamStatus: error.upstreamStatus,
upstreamBody: error.upstreamBody,
upstreamBodyTruncated: error.upstreamBodyTruncated,
}
: {}),
message,
messageLength: rawMessage.length,
messageTruncated: message.length !== rawMessage.length,
@ -2378,6 +2385,7 @@ async function handleWorkspaceFileRead(
content: numbered,
};
} catch (error) {
if (error instanceof WorkspaceToolHttpError) throw error;
if (signal?.aborted === true && isAbortError(error)) throw error;
logger.warn(
'[handleWorkspaceFileRead] Attached workspace read failed',
@ -2464,6 +2472,7 @@ async function handleWorkspaceSearchCall(
content: truncated ? `${content}${truncationNotice}` : content,
};
} catch (error) {
if (error instanceof WorkspaceToolHttpError) throw error;
if (signal?.aborted === true && isAbortError(error)) throw error;
logger.warn(
'[handleWorkspaceSearchCall] Attached workspace search failed',
@ -2566,6 +2575,7 @@ async function handleWorkspaceListCall(
content: `${content}${truncationNotice}`,
};
} catch (error) {
if (error instanceof WorkspaceToolHttpError) throw error;
if (signal?.aborted === true && isAbortError(error)) throw error;
logger.warn(
'[handleWorkspaceListCall] Attached workspace file listing failed',
@ -3725,8 +3735,11 @@ async function handleAttachedWorkspaceCreateFileCall({
created: result.created,
});
} catch (error) {
if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 409 && !overwrite) {
return errorResult(tc, 'File already exists. Pass overwrite: true to replace.');
if (error instanceof WorkspaceToolHttpError) {
if (error.upstreamStatus === 409 && !overwrite) {
error.message += '. File already exists. Pass overwrite: true to replace.';
}
throw error;
}
if (signal?.aborted === true && isAbortError(error)) throw error;
logger.warn('[file_authoring] Attached workspace write failed', getSafeErrorMetadata(error));
@ -3798,10 +3811,8 @@ async function handleAttachedWorkspaceEditFileCall({
} catch (error) {
if (signal?.aborted === true && isAbortError(error)) throw error;
if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 400) {
return errorResult(
tc,
'This attached environment must update its LibreChat Code worker before protected files can be edited.',
);
error.message +=
'. This attached environment must update its LibreChat Code worker before protected files can be edited.';
}
throw error;
}
@ -3828,11 +3839,11 @@ async function handleAttachedWorkspaceEditFileCall({
},
);
} catch (error) {
if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 409) {
return errorResult(
tc,
`The requested text did not match exactly once in "workspace/${path.filePath}". Re-read the file and retry.`,
);
if (error instanceof WorkspaceToolHttpError) {
if (error.upstreamStatus === 409) {
error.message += `; The requested text did not match exactly once in "workspace/${path.filePath}". Re-read the file and retry.`;
}
throw error;
}
if (signal?.aborted === true && isAbortError(error)) throw error;
logger.warn('[file_authoring] Attached workspace edit failed', getSafeErrorMetadata(error));
@ -5963,6 +5974,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
policyError == null
? filteredToolOutputResult(tc, backgroundReq, {
errorMessage: errorOutput,
upstreamBody:
toolError instanceof WorkspaceToolHttpError
? toolError.upstreamBody
: undefined,
})
: null;
const neutralizedError = filteredError?.errorMessage ?? errorOutput;
@ -6373,6 +6388,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const { message, logContext } = getSafeToolError(toolError);
const filteredError = filteredToolOutputResult(tc, req, {
errorMessage: message,
upstreamBody:
toolError instanceof WorkspaceToolHttpError
? toolError.upstreamBody
: undefined,
});
if (filteredError != null) {
logger.error(`[ON_TOOL_EXECUTE] Tool ${tc.name} error`, {
@ -6781,6 +6800,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const req = mergedConfigurable?.req as ServerRequest | undefined;
const filteredError = filteredToolOutputResult(tc, req, {
errorMessage: message,
upstreamBody:
toolError instanceof WorkspaceToolHttpError
? toolError.upstreamBody
: undefined,
});
if (filteredError != null) {
logToolFailure({

View file

@ -273,7 +273,7 @@ describe('executeWorkspaceTool', () => {
).rejects.toMatchObject({ reason: 'invalid' });
});
test('surfaces bounded upstream failures without returning their body', async () => {
test('preserves the upstream status when the error body stalls', async () => {
const cancel = jest.fn();
const fetchImpl = jest.fn().mockResolvedValue(
new Response(
@ -296,10 +296,105 @@ describe('executeWorkspaceTool', () => {
},
fetchImpl,
}),
).rejects.toMatchObject({ reason: 'rejected', upstreamStatus: 503 });
).rejects.toMatchObject({
reason: 'rejected',
upstreamStatus: 503,
upstreamBodyTruncated: true,
});
expect(cancel).toHaveBeenCalledTimes(1);
});
test.each([400, 409, 503, 504])('preserves HTTP %i and its diagnostic body', async (status) => {
const body = JSON.stringify({ code: 'ASSIGNMENT_EXPIRED', error: 'Assignment expired' });
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example.com',
authHeaders: {},
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
fetchImpl: jest.fn(async () => new Response(body, { status })),
}),
).rejects.toMatchObject({
reason: 'rejected',
upstreamStatus: status,
upstreamBody: body,
upstreamBodyTruncated: false,
message: expect.stringContaining(`upstreamStatus: ${status}`),
});
});
test.each([4095, 4096, 4097])(
'reports truncation correctly for a %i-byte error body',
async (size) => {
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example.com',
authHeaders: {},
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
fetchImpl: jest.fn(async () => new Response('x'.repeat(size), { status: 503 })),
}),
).rejects.toMatchObject({
upstreamStatus: 503,
upstreamBody: 'x'.repeat(Math.min(size, 4096)),
upstreamBodyTruncated: size > 4096,
});
},
);
test('preserves caller cancellation during a rejected response body read', async () => {
const controller = new AbortController();
const cancel = jest.fn();
const request = executeWorkspaceTool({
baseURL: 'https://code.example.com',
authHeaders: {},
signal: controller.signal,
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
fetchImpl: jest.fn(async () => new Response(new ReadableStream({ cancel }), { status: 503 })),
});
await new Promise<void>((resolve) => setImmediate(resolve));
controller.abort();
await expect(request).rejects.toBe(controller.signal.reason);
expect(cancel).toHaveBeenCalledTimes(1);
});
test('bounds a streaming error body and cancels the unread remainder', async () => {
const cancel = jest.fn();
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode('x'.repeat(10_000)));
},
cancel,
});
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example.com',
authHeaders: {},
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
fetchImpl: jest.fn(async () => new Response(body, { status: 504 })),
}),
).rejects.toMatchObject({
upstreamStatus: 504,
upstreamBody: 'x'.repeat(4096),
upstreamBodyTruncated: true,
});
expect(cancel).toHaveBeenCalledTimes(1);
});
test('retains HTTP status when reading the error body fails', async () => {
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.error(new Error('socket closed'));
},
});
await expect(
executeWorkspaceTool({
baseURL: 'https://code.example.com',
authHeaders: {},
request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' },
fetchImpl: jest.fn(async () => new Response(body, { status: 503 })),
}),
).rejects.toMatchObject({ upstreamStatus: 503, upstreamBodyTruncated: true });
});
test('validates bounded search matches before returning them', async () => {
const fetchImpl = jest.fn().mockResolvedValue(
new Response(

View file

@ -18,6 +18,8 @@ const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024;
const MAX_COMMAND_SIGNAL_LENGTH = 32;
const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000;
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
const MAX_ERROR_BODY_BYTES = 4096;
const ERROR_BODY_TIMEOUT_MS = 1000;
const READ_RESULT_KEYS = new Set([
'protocolVersion',
'operation',
@ -246,12 +248,64 @@ export class WorkspaceToolHttpError extends Error {
constructor(
public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed',
public readonly upstreamStatus?: number,
public readonly upstreamBody?: string,
public readonly upstreamBodyTruncated = false,
) {
super(`Workspace tool request ${reason}`);
super(
`Workspace tool request ${reason}` +
(upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) +
(upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') +
(upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''),
);
this.name = 'WorkspaceToolHttpError';
}
}
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
async function readErrorBody(
response: Response,
signal: AbortSignal,
): Promise<{
body: string;
truncated: boolean;
}> {
if (!response.body) return { body: '', truncated: false };
const reader = response.body.getReader();
const decoder = new TextDecoder();
let body = '';
let bytes = 0;
let complete = false;
let interrupted = false;
const cancel = () => {
interrupted = true;
void reader.cancel().catch(() => undefined);
};
const timer = setTimeout(cancel, ERROR_BODY_TIMEOUT_MS);
signal.addEventListener('abort', cancel, { once: true });
try {
if (signal.aborted) return { body, truncated: true };
while (bytes <= MAX_ERROR_BODY_BYTES) {
const { done, value } = await reader.read();
if (done) {
complete = !interrupted;
body += decoder.decode();
break;
}
const remaining = MAX_ERROR_BODY_BYTES - bytes;
body += decoder.decode(value.subarray(0, remaining), { stream: true });
bytes += value.byteLength;
}
} catch {
complete = false;
} finally {
clearTimeout(timer);
signal.removeEventListener('abort', cancel);
cancel();
reader.releaseLock();
}
return { body, truncated: !complete || signal.aborted };
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value != null;
}
@ -639,8 +693,9 @@ export async function executeWorkspaceTool({
},
);
if (!response.ok) {
await response.body?.cancel().catch(() => undefined);
throw new WorkspaceToolHttpError('rejected', response.status);
const { body, truncated } = await readErrorBody(response, requestSignal);
signal?.throwIfAborted();
throw new WorkspaceToolHttpError('rejected', response.status, body, truncated);
}
const result = await readBoundedJson(response, requestSignal);
if (!isValidResult(request, result)) {