diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts index 028fa9829b..f8a0fc19c7 100644 --- a/packages/api/src/agents/handlers.spec.ts +++ b/packages/api/src/agents/handlers.spec.ts @@ -1459,58 +1459,92 @@ describe('createToolExecuteHandler', () => { } }); - it('filters thrown foreground errors before result delivery or logging', async () => { - const protectedValue = 'PROTECTED-FOREGROUND-ERROR'; - const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({ - loadedTools: [ - { - name: 'throwing_tool', - invoke: jest.fn(async () => { - throw new Error(protectedValue); - }), - }, - ] as never[], - })); - const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger); - try { - const handler = createToolExecuteHandler({ loadTools }); - const [result] = await invokeHandlerWithConfig( - handler, - [{ id: 'call_filtered_throw', name: 'throwing_tool', args: {} }], - { - req: { - config: { - filters: { - toolArguments: { - pii: { - fields: ['output'], - starterPatterns: [], - customPatterns: [ - { - id: 'protected-output', - label: 'protected output', - regex: 'PROTECTED-[A-Z-]+', - }, - ], + it.each(['generic', 'workspace', 'workspace-expanded'])( + 'filters %s foreground errors before result delivery or logging', + async (kind) => { + const protectedValue = 'PROTECTED-FOREGROUND-ERROR'; + const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({ + loadedTools: [ + { + name: 'throwing_tool', + invoke: jest.fn(async () => { + if (kind.startsWith('workspace')) { + const body = + kind === 'workspace-expanded' + ? '\u0001'.repeat(2000) + protectedValue + '\u0001'.repeat(2000) + : protectedValue; + throw new WorkspaceToolHttpError('rejected', 503, body); + } + throw new Error(protectedValue); + }), + }, + ] as never[], + })); + const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger); + try { + const handler = createToolExecuteHandler({ loadTools }); + const [result] = await invokeHandlerWithConfig( + handler, + [{ id: 'call_filtered_throw', name: 'throwing_tool', args: {} }], + { + req: { + config: { + filters: { + toolArguments: { + pii: { + fields: ['output'], + starterPatterns: [], + customPatterns: [ + { + id: 'protected-output', + label: 'protected output', + regex: 'PROTECTED-[A-Z-]+', + }, + ], + }, }, }, }, }, }, - }, - ); + ); - expect(result.status).toBe('error'); - expect(result.errorMessage).toContain('content_filter_block'); - expect(result.errorMessage).not.toContain(protectedValue); - expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(protectedValue); - expect(errorSpy).toHaveBeenCalledWith( - '[ON_TOOL_EXECUTE] Tool throwing_tool error', - expect.objectContaining({ contentFiltered: true }), - ); - } finally { - errorSpy.mockRestore(); - } + expect(result.status).toBe('error'); + expect(result.errorMessage).toContain('content_filter_block'); + expect(result.errorMessage).not.toContain(protectedValue); + expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(protectedValue); + expect(errorSpy).toHaveBeenCalledWith( + '[ON_TOOL_EXECUTE] Tool throwing_tool error', + expect.objectContaining({ contentFiltered: true }), + ); + } finally { + errorSpy.mockRestore(); + } + }, + ); + + it('surfaces workspace transport diagnostics thrown by a loaded tool', async () => { + const body = '{"code":"ASSIGNMENT_EXPIRED"}'; + const loadTools: ToolExecuteOptions['loadTools'] = jest.fn(async () => ({ + loadedTools: [ + { + name: 'workspace_command', + invoke: jest.fn(async () => { + throw new WorkspaceToolHttpError('rejected', 504, body); + }), + }, + ] as never[], + })); + const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger); + const [result] = await invokeHandler(createToolExecuteHandler({ loadTools }), [ + { id: 'call_command_error', name: 'workspace_command', args: {} }, + ]); + expect(result.errorMessage).toContain('upstreamStatus: 504'); + expect(result.errorMessage).toContain('ASSIGNMENT_EXPIRED'); + expect(errorSpy).toHaveBeenCalledWith( + '[ON_TOOL_EXECUTE] Tool workspace_command error', + expect.objectContaining({ upstreamStatus: 504, upstreamBody: body }), + ); }); it('truncates oversized tool errors in the result and log context', async () => { @@ -5524,6 +5558,42 @@ describe('createToolExecuteHandler', () => { expect(searchWorkspace).not.toHaveBeenCalled(); }); + it.each([400, 409, 503, 504])( + 'surfaces workspace HTTP %i in logs and model results', + async (status) => { + const body = '{"code":"WORKER_BUSY","error":"Worker is busy"}'; + const errorSpy = jest.spyOn(logger, 'error').mockReturnValue(logger); + const handler = makeReadFileHandler({ + codeEnvAvailable: true, + codeExecutionContext: { + baseUrl: 'https://code.example.com', + codeSessionKey: 'attached', + executionProfile: 'stateful', + statefulSessions: true, + environmentType: 'attached', + }, + listWorkspaceFiles: jest.fn(async () => { + throw new WorkspaceToolHttpError('rejected', status, body); + }), + }); + const [result] = await invokeHandler(handler, [ + { id: 'call_workspace_error', name: 'list_workspace_files', args: {} }, + ]); + expect(result.status).toBe('error'); + expect(result.errorMessage).toContain(`upstreamStatus: ${status}`); + expect(result.errorMessage).toContain('WORKER_BUSY'); + expect(errorSpy).toHaveBeenCalledWith( + '[ON_TOOL_EXECUTE] Tool list_workspace_files error', + expect.objectContaining({ + name: 'WorkspaceToolHttpError', + upstreamStatus: status, + upstreamBody: body, + upstreamBodyTruncated: false, + }), + ); + }, + ); + it('lists files through the selected attached worker and forwards cancellation', async () => { const controller = new AbortController(); const listWorkspaceFiles = jest.fn(async () => ({ diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 7802e2611b..5dc5a3ae70 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -765,6 +765,13 @@ function getSafeToolError(error: unknown): { message, logContext: { name: error instanceof Error ? error.name : typeof error, + ...(error instanceof WorkspaceToolHttpError + ? { + upstreamStatus: error.upstreamStatus, + upstreamBody: error.upstreamBody, + upstreamBodyTruncated: error.upstreamBodyTruncated, + } + : {}), message, messageLength: rawMessage.length, messageTruncated: message.length !== rawMessage.length, @@ -2378,6 +2385,7 @@ async function handleWorkspaceFileRead( content: numbered, }; } catch (error) { + if (error instanceof WorkspaceToolHttpError) throw error; if (signal?.aborted === true && isAbortError(error)) throw error; logger.warn( '[handleWorkspaceFileRead] Attached workspace read failed', @@ -2464,6 +2472,7 @@ async function handleWorkspaceSearchCall( content: truncated ? `${content}${truncationNotice}` : content, }; } catch (error) { + if (error instanceof WorkspaceToolHttpError) throw error; if (signal?.aborted === true && isAbortError(error)) throw error; logger.warn( '[handleWorkspaceSearchCall] Attached workspace search failed', @@ -2566,6 +2575,7 @@ async function handleWorkspaceListCall( content: `${content}${truncationNotice}`, }; } catch (error) { + if (error instanceof WorkspaceToolHttpError) throw error; if (signal?.aborted === true && isAbortError(error)) throw error; logger.warn( '[handleWorkspaceListCall] Attached workspace file listing failed', @@ -3725,8 +3735,11 @@ async function handleAttachedWorkspaceCreateFileCall({ created: result.created, }); } catch (error) { - if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 409 && !overwrite) { - return errorResult(tc, 'File already exists. Pass overwrite: true to replace.'); + if (error instanceof WorkspaceToolHttpError) { + if (error.upstreamStatus === 409 && !overwrite) { + error.message += '. File already exists. Pass overwrite: true to replace.'; + } + throw error; } if (signal?.aborted === true && isAbortError(error)) throw error; logger.warn('[file_authoring] Attached workspace write failed', getSafeErrorMetadata(error)); @@ -3798,10 +3811,8 @@ async function handleAttachedWorkspaceEditFileCall({ } catch (error) { if (signal?.aborted === true && isAbortError(error)) throw error; if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 400) { - return errorResult( - tc, - 'This attached environment must update its LibreChat Code worker before protected files can be edited.', - ); + error.message += + '. This attached environment must update its LibreChat Code worker before protected files can be edited.'; } throw error; } @@ -3828,11 +3839,11 @@ async function handleAttachedWorkspaceEditFileCall({ }, ); } catch (error) { - if (error instanceof WorkspaceToolHttpError && error.upstreamStatus === 409) { - return errorResult( - tc, - `The requested text did not match exactly once in "workspace/${path.filePath}". Re-read the file and retry.`, - ); + if (error instanceof WorkspaceToolHttpError) { + if (error.upstreamStatus === 409) { + error.message += `; The requested text did not match exactly once in "workspace/${path.filePath}". Re-read the file and retry.`; + } + throw error; } if (signal?.aborted === true && isAbortError(error)) throw error; logger.warn('[file_authoring] Attached workspace edit failed', getSafeErrorMetadata(error)); @@ -5963,6 +5974,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand policyError == null ? filteredToolOutputResult(tc, backgroundReq, { errorMessage: errorOutput, + upstreamBody: + toolError instanceof WorkspaceToolHttpError + ? toolError.upstreamBody + : undefined, }) : null; const neutralizedError = filteredError?.errorMessage ?? errorOutput; @@ -6373,6 +6388,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand const { message, logContext } = getSafeToolError(toolError); const filteredError = filteredToolOutputResult(tc, req, { errorMessage: message, + upstreamBody: + toolError instanceof WorkspaceToolHttpError + ? toolError.upstreamBody + : undefined, }); if (filteredError != null) { logger.error(`[ON_TOOL_EXECUTE] Tool ${tc.name} error`, { @@ -6781,6 +6800,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand const req = mergedConfigurable?.req as ServerRequest | undefined; const filteredError = filteredToolOutputResult(tc, req, { errorMessage: message, + upstreamBody: + toolError instanceof WorkspaceToolHttpError + ? toolError.upstreamBody + : undefined, }); if (filteredError != null) { logToolFailure({ diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts index 63f1b6ab98..14a2ccd42a 100644 --- a/packages/api/src/code/workspace.spec.ts +++ b/packages/api/src/code/workspace.spec.ts @@ -273,7 +273,7 @@ describe('executeWorkspaceTool', () => { ).rejects.toMatchObject({ reason: 'invalid' }); }); - test('surfaces bounded upstream failures without returning their body', async () => { + test('preserves the upstream status when the error body stalls', async () => { const cancel = jest.fn(); const fetchImpl = jest.fn().mockResolvedValue( new Response( @@ -296,10 +296,105 @@ describe('executeWorkspaceTool', () => { }, fetchImpl, }), - ).rejects.toMatchObject({ reason: 'rejected', upstreamStatus: 503 }); + ).rejects.toMatchObject({ + reason: 'rejected', + upstreamStatus: 503, + upstreamBodyTruncated: true, + }); expect(cancel).toHaveBeenCalledTimes(1); }); + test.each([400, 409, 503, 504])('preserves HTTP %i and its diagnostic body', async (status) => { + const body = JSON.stringify({ code: 'ASSIGNMENT_EXPIRED', error: 'Assignment expired' }); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com', + authHeaders: {}, + request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + fetchImpl: jest.fn(async () => new Response(body, { status })), + }), + ).rejects.toMatchObject({ + reason: 'rejected', + upstreamStatus: status, + upstreamBody: body, + upstreamBodyTruncated: false, + message: expect.stringContaining(`upstreamStatus: ${status}`), + }); + }); + + test.each([4095, 4096, 4097])( + 'reports truncation correctly for a %i-byte error body', + async (size) => { + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com', + authHeaders: {}, + request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + fetchImpl: jest.fn(async () => new Response('x'.repeat(size), { status: 503 })), + }), + ).rejects.toMatchObject({ + upstreamStatus: 503, + upstreamBody: 'x'.repeat(Math.min(size, 4096)), + upstreamBodyTruncated: size > 4096, + }); + }, + ); + + test('preserves caller cancellation during a rejected response body read', async () => { + const controller = new AbortController(); + const cancel = jest.fn(); + const request = executeWorkspaceTool({ + baseURL: 'https://code.example.com', + authHeaders: {}, + signal: controller.signal, + request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + fetchImpl: jest.fn(async () => new Response(new ReadableStream({ cancel }), { status: 503 })), + }); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(); + await expect(request).rejects.toBe(controller.signal.reason); + expect(cancel).toHaveBeenCalledTimes(1); + }); + + test('bounds a streaming error body and cancels the unread remainder', async () => { + const cancel = jest.fn(); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('x'.repeat(10_000))); + }, + cancel, + }); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com', + authHeaders: {}, + request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + fetchImpl: jest.fn(async () => new Response(body, { status: 504 })), + }), + ).rejects.toMatchObject({ + upstreamStatus: 504, + upstreamBody: 'x'.repeat(4096), + upstreamBodyTruncated: true, + }); + expect(cancel).toHaveBeenCalledTimes(1); + }); + + test('retains HTTP status when reading the error body fails', async () => { + const body = new ReadableStream({ + start(controller) { + controller.error(new Error('socket closed')); + }, + }); + await expect( + executeWorkspaceTool({ + baseURL: 'https://code.example.com', + authHeaders: {}, + request: { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + fetchImpl: jest.fn(async () => new Response(body, { status: 503 })), + }), + ).rejects.toMatchObject({ upstreamStatus: 503, upstreamBodyTruncated: true }); + }); + test('validates bounded search matches before returning them', async () => { const fetchImpl = jest.fn().mockResolvedValue( new Response( diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts index 84d275bb7e..633c266272 100644 --- a/packages/api/src/code/workspace.ts +++ b/packages/api/src/code/workspace.ts @@ -18,6 +18,8 @@ const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024; const MAX_COMMAND_SIGNAL_LENGTH = 32; const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000; const MAX_RESPONSE_BYTES = 4 * 1024 * 1024; +const MAX_ERROR_BODY_BYTES = 4096; +const ERROR_BODY_TIMEOUT_MS = 1000; const READ_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -246,12 +248,64 @@ export class WorkspaceToolHttpError extends Error { constructor( public readonly reason: 'rejected' | 'invalid' | 'timeout' | 'failed', public readonly upstreamStatus?: number, + public readonly upstreamBody?: string, + public readonly upstreamBodyTruncated = false, ) { - super(`Workspace tool request ${reason}`); + super( + `Workspace tool request ${reason}` + + (upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) + + (upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') + + (upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''), + ); this.name = 'WorkspaceToolHttpError'; } } +/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */ +async function readErrorBody( + response: Response, + signal: AbortSignal, +): Promise<{ + body: string; + truncated: boolean; +}> { + if (!response.body) return { body: '', truncated: false }; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let body = ''; + let bytes = 0; + let complete = false; + let interrupted = false; + const cancel = () => { + interrupted = true; + void reader.cancel().catch(() => undefined); + }; + const timer = setTimeout(cancel, ERROR_BODY_TIMEOUT_MS); + signal.addEventListener('abort', cancel, { once: true }); + try { + if (signal.aborted) return { body, truncated: true }; + while (bytes <= MAX_ERROR_BODY_BYTES) { + const { done, value } = await reader.read(); + if (done) { + complete = !interrupted; + body += decoder.decode(); + break; + } + const remaining = MAX_ERROR_BODY_BYTES - bytes; + body += decoder.decode(value.subarray(0, remaining), { stream: true }); + bytes += value.byteLength; + } + } catch { + complete = false; + } finally { + clearTimeout(timer); + signal.removeEventListener('abort', cancel); + cancel(); + reader.releaseLock(); + } + return { body, truncated: !complete || signal.aborted }; +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value != null; } @@ -639,8 +693,9 @@ export async function executeWorkspaceTool({ }, ); if (!response.ok) { - await response.body?.cancel().catch(() => undefined); - throw new WorkspaceToolHttpError('rejected', response.status); + const { body, truncated } = await readErrorBody(response, requestSignal); + signal?.throwIfAborted(); + throw new WorkspaceToolHttpError('rejected', response.status, body, truncated); } const result = await readBoundedJson(response, requestSignal); if (!isValidResult(request, result)) {