diff --git a/librechat.example.yaml b/librechat.example.yaml index 260912aa78..6c1cba99c0 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -587,6 +587,11 @@ endpoints: # # (optional) Assistant Capabilities available to all users. Omit the ones you wish to exclude. Defaults to list below. # capabilities: ["code_interpreter", "retrieval", "actions", "tools", "image_vision"] # agents: + # # Maximum provider response-body inactivity for Agent model calls. Defaults to 15 minutes; + # # each received chunk resets the timer. Set to 0 to disable the idle timeout. + # modelResponseBodyTimeoutMs: 900000 + # # Maximum wait for provider response headers. Defaults to 5 minutes; 0 disables. + # modelResponseHeadersTimeoutMs: 300000 # # (optional) Default recursion depth for agents, defaults to 25 # recursionLimit: 50 # # Maximum provider error characters retained without content protection (0 hides detail). diff --git a/packages/api/src/agents/__tests__/run-summarization.test.ts b/packages/api/src/agents/__tests__/run-summarization.test.ts index ede0868eb4..3b7e781e1c 100644 --- a/packages/api/src/agents/__tests__/run-summarization.test.ts +++ b/packages/api/src/agents/__tests__/run-summarization.test.ts @@ -2583,10 +2583,11 @@ describe('built-in provider request shaping', () => { const summarizeWith = async ( parameters?: Record, model = 'gpt-6-astra', + appConfig = makeAppConfig([]), ): Promise> => { const agents = await callAndCapture({ agents: [anthropicAgent()], - appConfig: makeAppConfig([]), + appConfig, summarizationConfig: { provider: 'openAI', model, @@ -2621,24 +2622,50 @@ describe('built-in provider request shaping', () => { }); it('does not claim a first-party endpoint behind a user configuration.baseURL', async () => { - expect( - await summarizeWith({ configuration: { baseURL: 'https://gateway.internal/v1' } }), - ).toEqual({ configuration: { baseURL: 'https://gateway.internal/v1' } }); + const parameters = await summarizeWith({ + configuration: { baseURL: 'https://gateway.internal/v1' }, + }); + expect(parameters.firstPartyEndpoint).toBeUndefined(); + expect(parameters.configuration).toMatchObject({ + baseURL: 'https://gateway.internal/v1', + fetchOptions: { dispatcher: expect.any(Object) }, + }); }); it('does not claim a first-party endpoint behind a user baseURL', async () => { expect(await summarizeWith({ baseURL: 'https://gateway.internal/v1' })).toEqual({ baseURL: 'https://gateway.internal/v1', + configuration: { fetchOptions: { dispatcher: expect.any(Object) } }, }); }); - it('leaves credentials and transport to the client', async () => { + it('adds only transport policy while leaving credentials and model selection to the client', async () => { const parameters = await summarizeWith(); expect(parameters.apiKey).toBeUndefined(); expect(parameters.model).toBeUndefined(); expect(parameters.modelName).toBeUndefined(); expect(parameters.streaming).toBeUndefined(); - expect(parameters.configuration).toBeUndefined(); + expect(parameters.configuration).toEqual({ + fetchOptions: { dispatcher: expect.any(Object) }, + }); + }); + + it.each([ + { bodyTimeout: 900_000, headersTimeout: 300_000 }, + { bodyTimeout: 1_800_000, headersTimeout: 120_000 }, + { bodyTimeout: 0, headersTimeout: 0 }, + ])('forwards the cross-provider timeout policy %j', async (transportTimeouts) => { + const appConfig = makeAppConfig([]); + appConfig.endpoints!.agents = { + modelResponseBodyTimeoutMs: transportTimeouts.bodyTimeout, + modelResponseHeadersTimeoutMs: transportTimeouts.headersTimeout, + }; + const parameters = await summarizeWith(undefined, 'gpt-4o', appConfig); + const expected = getOpenAIConfig('unused', { transportTimeouts }); + const configuration = parameters.configuration as NonNullable; + expect(configuration.fetchOptions?.dispatcher).toBe( + expected.configOptions?.fetchOptions?.dispatcher, + ); }); it('leaves a same-endpoint summarizer on the agent client options', async () => { @@ -2663,7 +2690,9 @@ describe('built-in provider request shaping', () => { it('withholds the declaration when a reverse proxy serves the built-in endpoint', async () => { process.env.OPENAI_REVERSE_PROXY = 'https://gateway.internal/v1'; try { - expect(await summarizeWith()).toBeUndefined(); + expect(await summarizeWith()).toEqual({ + configuration: { fetchOptions: { dispatcher: expect.any(Object) } }, + }); } finally { delete process.env.OPENAI_REVERSE_PROXY; } @@ -2672,7 +2701,9 @@ describe('built-in provider request shaping', () => { it('withholds the declaration when the base URL is user-provided', async () => { process.env.OPENAI_REVERSE_PROXY = 'user_provided'; try { - expect(await summarizeWith()).toBeUndefined(); + expect(await summarizeWith()).toEqual({ + configuration: { fetchOptions: { dispatcher: expect.any(Object) } }, + }); } finally { delete process.env.OPENAI_REVERSE_PROXY; } diff --git a/packages/api/src/agents/config.spec.ts b/packages/api/src/agents/config.spec.ts index af98cf04b5..1c6ee4c9af 100644 --- a/packages/api/src/agents/config.spec.ts +++ b/packages/api/src/agents/config.spec.ts @@ -1,5 +1,10 @@ import type { TAgentsEndpoint } from 'librechat-data-provider'; -import { resolveRecursionLimit, resolveStreamLimits, resolveSubagentMaxTurns } from './config'; +import { + resolveStreamLimits, + resolveRecursionLimit, + resolveSubagentMaxTurns, + resolveModelTransportTimeouts, +} from './config'; describe('resolveRecursionLimit', () => { it('returns default 50 when no config or agent provided', () => { @@ -112,6 +117,21 @@ describe('resolveSubagentMaxTurns', () => { }); }); +describe('resolveModelTransportTimeouts', () => { + it('uses finite defaults and preserves explicit overrides including zero', () => { + expect(resolveModelTransportTimeouts(undefined)).toEqual({ + bodyTimeout: 900_000, + headersTimeout: 300_000, + }); + expect( + resolveModelTransportTimeouts({ + modelResponseBodyTimeoutMs: 1_800_000, + modelResponseHeadersTimeoutMs: 0, + }), + ).toEqual({ bodyTimeout: 1_800_000, headersTimeout: 0 }); + }); +}); + describe('resolveStreamLimits', () => { const CREATE_FILE_DEFAULT = { create_file: 131072 }; diff --git a/packages/api/src/agents/config.ts b/packages/api/src/agents/config.ts index df413e69bc..62f28f3d5f 100644 --- a/packages/api/src/agents/config.ts +++ b/packages/api/src/agents/config.ts @@ -1,8 +1,29 @@ +import { + DEFAULT_AGENT_MODEL_RESPONSE_BODY_TIMEOUT_MS, + DEFAULT_AGENT_MODEL_RESPONSE_HEADERS_TIMEOUT_MS, +} from 'librechat-data-provider'; import type { TAgentsEndpoint } from 'librechat-data-provider'; import { CREATE_FILE_TOOL_NAME } from '~/agents/tools'; const DEFAULT_RECURSION_LIMIT = 50; +export interface ModelTransportTimeouts { + bodyTimeout: number; + headersTimeout: number; +} + +export function resolveModelTransportTimeouts( + agentsEConfig: Partial | undefined, +): ModelTransportTimeouts { + return { + bodyTimeout: + agentsEConfig?.modelResponseBodyTimeoutMs ?? DEFAULT_AGENT_MODEL_RESPONSE_BODY_TIMEOUT_MS, + headersTimeout: + agentsEConfig?.modelResponseHeadersTimeoutMs ?? + DEFAULT_AGENT_MODEL_RESPONSE_HEADERS_TIMEOUT_MS, + }; +} + /** * Mirrors `RECURSION_MULTIPLIER` in `@librechat/agents` `SubagentExecutor`, * which derives a subagent's graph `recursionLimit` as `maxTurns * 3`. Keep in diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index c6287521cc..27a00ff223 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -82,6 +82,12 @@ import { agentUsesSubagentCompletionWakeups, usesSubagentCompletionWakeups, } from '~/agents/subagentDelivery'; +import { + resolveStreamLimits, + resolveModelTransportTimeouts, + resolveSubagentMaxTurns, + resolveRecursionLimit, +} from '~/agents/config'; import { isSteeringSupported, isSteerPreemptSupported, @@ -101,11 +107,6 @@ import { eventOnlyRunFileTools, isRunFileSharingSupported, } from './files/runtime'; -import { - resolveStreamLimits, - resolveSubagentMaxTurns, - resolveRecursionLimit, -} from '~/agents/config'; import { applyCustomHandoffPromptKeyCompatibility } from '~/agents/handoffPromptKeyCompatibility'; import { stripIntentFromToolRegistry, stripIntentFromToolDefinitions } from '~/agents/intent'; import { resolveConfigHeaders, resolveModelHeaders, mergeHeaders } from '~/utils/headers'; @@ -113,6 +114,7 @@ import { extractDefaultParams, resolveReasoningParams } from '~/endpoints/openai import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm'; import { CREATE_FILE_TOOL_NAME, EDIT_FILE_TOOL_NAME } from '~/agents/tools'; import { buildAgentInitialToolSessions } from '~/agents/codeFilesSession'; +import { getDirectDispatcher, getProxyDispatcher } from '~/utils/proxy'; import { getAzureCredentials, constructAzureURL } from '~/utils/azure'; import { getBuiltInBaseURL } from '~/endpoints/openai/initialize'; import { getProviderConfig } from '~/endpoints/config/providers'; @@ -701,11 +703,9 @@ function summarizationReasoningEffort( * defaults its model-specific constraints off without that declaration * (LibreChat#15598). * - * Credentials and transport are deliberately not returned. A built-in provider - * has no configured key here, so the client resolves one the way it does today; - * emitting an empty `apiKey` would break that. The admin-configured base URL is - * still passed *in*, because whether the endpoint is first-party is exactly what - * `OPENAI_REVERSE_PROXY` decides. + * Credentials and base URLs are not returned: the SDK still resolves them as + * before. Cross-provider OpenAI-family clients do receive the Agent transport + * timeout policy, even when a URL override prevents built-in request shaping. */ function resolveBuiltInClientOverrides( provider: string, @@ -714,11 +714,9 @@ function resolveBuiltInClientOverrides( parameters?: SummarizationConfig['parameters']; agentProvider?: string; }, + appConfig: AppConfig, ): SummarizationClientOverrides | undefined { const { model, parameters } = target; - if (!isNonEmptyString(model) || hasBaseURLOverride(parameters)) { - return undefined; - } /** * Mirrors the SDK's own condition: when the summarization provider matches the * agent's, `buildSummarizationClientConfig` spreads the agent's resolved client @@ -730,10 +728,22 @@ function resolveBuiltInClientOverrides( if (provider === target.agentProvider) { return undefined; } + let transportOverrides: SummarizationClientOverrides | undefined; + if (provider === Providers.OPENAI || provider === Providers.AZURE) { + const timeouts = resolveModelTransportTimeouts(appConfig.endpoints?.agents); + transportOverrides = { + configuration: { + fetchOptions: { + dispatcher: + getProxyDispatcher(process.env.PROXY, timeouts) ?? getDirectDispatcher(timeouts), + }, + }, + }; + } const baseURL = getBuiltInBaseURL(provider); - /** Resolving a user-provided base URL needs a database read this path avoids. */ - if (isUserProvided(baseURL)) { - return undefined; + /** URL overrides still get timeouts, but must not inherit first-party request shaping. */ + if (!isNonEmptyString(model) || hasBaseURLOverride(parameters) || isUserProvided(baseURL)) { + return transportOverrides; } const { llmConfig } = getOpenAIConfig( '', @@ -750,7 +760,9 @@ function resolveBuiltInClientOverrides( streaming: _streaming, ...shaping } = llmConfig; - return Object.keys(shaping).length > 0 ? shaping : undefined; + return Object.keys(shaping).length > 0 + ? { ...shaping, ...transportOverrides } + : transportOverrides; } /** @@ -906,6 +918,7 @@ function resolveOpenAISummarization( tenantId: headerContext.tenantId, body: headerContext.requestBody, }), + transportTimeouts: resolveModelTransportTimeouts(appConfig?.endpoints?.agents), }, EModelEndpoint.openAI, ); @@ -1077,6 +1090,7 @@ function resolveAzureSummarization( dropParams: group?.dropParams?.filter( (key) => key !== 'useResponsesApi' || typeof parameters?.useResponsesApi !== 'boolean', ), + transportTimeouts: resolveModelTransportTimeouts(appConfig?.endpoints?.agents), }, EModelEndpoint.azureOpenAI, ); @@ -1150,7 +1164,7 @@ function resolveSummarizationProvider( if (!customEndpointConfig) { return { provider: overrideProvider, - clientOverrides: resolveBuiltInClientOverrides(overrideProvider, target), + clientOverrides: resolveBuiltInClientOverrides(overrideProvider, target, appConfig), }; } const rawApiKey = customEndpointConfig.apiKey ?? ''; @@ -1251,6 +1265,7 @@ function resolveSummarizationProvider( dropParams: customEndpointConfig.dropParams, customParams: customEndpointConfig.customParams, directEndpoint: customEndpointConfig.directEndpoint, + transportTimeouts: resolveModelTransportTimeouts(appConfig?.endpoints?.agents), }, rawProvider, ); diff --git a/packages/api/src/endpoints/custom/initialize.spec.ts b/packages/api/src/endpoints/custom/initialize.spec.ts index d52d2aea74..c76c84af31 100644 --- a/packages/api/src/endpoints/custom/initialize.spec.ts +++ b/packages/api/src/endpoints/custom/initialize.spec.ts @@ -199,6 +199,29 @@ describe('initializeCustom – OpenAI-compatible header forwarding', () => { jest.clearAllMocks(); }); + it('forwards Agent model transport timeout policy', async () => { + const params = createParams({ + apiKey: 'sk-system-key', + baseURL: 'https://gateway.example.com/v1', + }); + (params.req.config as { endpoints: Record }).endpoints = { + agents: { + modelResponseBodyTimeoutMs: 1_800_000, + modelResponseHeadersTimeoutMs: 120_000, + }, + }; + + await initializeCustom(params); + + const clientOptions = mockGetOpenAIConfig.mock.calls[0][1] as { + transportTimeouts?: { bodyTimeout: number; headersTimeout: number }; + }; + expect(clientOptions.transportTimeouts).toEqual({ + bodyTimeout: 1_800_000, + headersTimeout: 120_000, + }); + }); + it('preserves configured headers for admin-trusted base URLs', async () => { const headers = { Authorization: 'Bearer static-gateway-token', diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.ts index 8966d4c9c2..e254c9543a 100644 --- a/packages/api/src/endpoints/custom/initialize.ts +++ b/packages/api/src/endpoints/custom/initialize.ts @@ -15,6 +15,7 @@ import type { ProviderInitializeParams, } from '~/types'; import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm'; +import { resolveModelTransportTimeouts } from '~/agents/config'; import { extractDefaultParams } from '~/endpoints/openai/llm'; import { isUserProvided, checkUserKeyExpiry } from '~/utils'; import { getOpenAIConfig } from '~/endpoints/openai/config'; @@ -311,6 +312,7 @@ export async function initializeCustom( reverseProxyUrl: baseURL ?? null, baseURLIsUserProvided: userProvidesURL, allowedAddresses: appConfig?.endpoints?.allowedAddresses, + transportTimeouts: resolveModelTransportTimeouts(appConfig?.endpoints?.agents), proxy: PROXY ?? null, ...customOptions, }; diff --git a/packages/api/src/endpoints/openai/config.spec.ts b/packages/api/src/endpoints/openai/config.spec.ts index 4b1300e51e..951ab72b74 100644 --- a/packages/api/src/endpoints/openai/config.spec.ts +++ b/packages/api/src/endpoints/openai/config.spec.ts @@ -26,6 +26,14 @@ describe('getOpenAIConfig', () => { expect(result.tools).toEqual([]); }); + it('applies an explicit model transport timeout policy', () => { + const result = getOpenAIConfig(mockApiKey, { + transportTimeouts: { bodyTimeout: 900_000, headersTimeout: 300_000 }, + }); + + expect(result.configOptions?.fetchOptions?.dispatcher).toBeDefined(); + }); + it('should apply model options', () => { const modelOptions = { model: 'gpt-4', diff --git a/packages/api/src/endpoints/openai/config.ts b/packages/api/src/endpoints/openai/config.ts index 74aeb1118d..46067e9517 100644 --- a/packages/api/src/endpoints/openai/config.ts +++ b/packages/api/src/endpoints/openai/config.ts @@ -1,15 +1,15 @@ -import { Agent } from 'undici'; import { Providers } from '@librechat/agents'; +import { Agent, fetch as undiciFetch } from 'undici'; import { KnownEndpoints, EModelEndpoint, ReasoningParameterFormat } from 'librechat-data-provider'; import type { Dispatcher } from 'undici'; import type * as t from '~/types'; import { getGoogleConfig, stripGeminiFlashBlockedParams } from '~/endpoints/google/llm'; import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm'; import { createSSRFSafeAgents, createSSRFSafeUndiciConnect } from '~/auth'; +import { getDirectDispatcher, getProxyDispatcher } from '~/utils/proxy'; import { getOpenAILLMConfig, extractDefaultParams } from './llm'; import { constructAzureResponsesURL } from '~/utils/azure'; import { transformToOpenAIConfig } from './transform'; -import { getProxyDispatcher } from '~/utils/proxy'; import { createFetch } from '~/utils/generators'; import { mergeHeaders } from '~/utils/headers'; @@ -247,9 +247,11 @@ export function getOpenAIConfig( configOptions.defaultQuery = defaultQuery; } + const transportTimeouts = options.transportTimeouts; if (shouldProtectUserBaseURL) { mergeFetchOptions(configOptions, { dispatcher: new Agent({ + ...transportTimeouts, connect: createSSRFSafeUndiciConnect( options.allowedAddresses, getEffectiveURLPort(baseURL), @@ -257,11 +259,16 @@ export function getOpenAIConfig( }), redirect: 'error', }); - } - - const proxyDispatcher = getProxyDispatcher(proxy); - if (proxyDispatcher && !shouldProtectUserBaseURL) { - mergeFetchOptions(configOptions, { dispatcher: proxyDispatcher }); + } else if (transportTimeouts != null) { + const proxyDispatcher = getProxyDispatcher(proxy, transportTimeouts); + mergeFetchOptions(configOptions, { + dispatcher: proxyDispatcher ?? getDirectDispatcher(transportTimeouts), + }); + } else { + const proxyDispatcher = getProxyDispatcher(proxy); + if (proxyDispatcher) { + mergeFetchOptions(configOptions, { dispatcher: proxyDispatcher }); + } } if (azure && !isAnthropic) { @@ -296,12 +303,23 @@ export function getOpenAIConfig( } if (directEndpoint === true && configOptions?.baseURL != null) { - configOptions.fetch = createFetch({ - directEndpoint: directEndpoint, - reverseProxyUrl: configOptions?.baseURL, - ssrfAgents, - redirect: shouldProtectUserBaseURL ? 'error' : undefined, - }) as unknown as Fetch; + const directURL = configOptions.baseURL; + if (transportTimeouts != null) { + /** Keep the exact URL, but use the same proxy/SSRF/timeout policy as SDK requests. */ + configOptions.fetch = ((_url: string | URL | Request, init?: RequestInit) => + undiciFetch(directURL, { + ...(init as Parameters[1]), + // These options are constructed above with an Undici dispatcher, not another SDK runtime. + ...(configOptions.fetchOptions as Parameters[1]), + })) as unknown as Fetch; + } else { + configOptions.fetch = createFetch({ + directEndpoint, + reverseProxyUrl: directURL, + ssrfAgents, + redirect: shouldProtectUserBaseURL ? 'error' : undefined, + }) as unknown as Fetch; + } } const result: t.OpenAIConfigResult = { diff --git a/packages/api/src/endpoints/openai/initialize.spec.ts b/packages/api/src/endpoints/openai/initialize.spec.ts index 0a1d65df66..5cf273bc29 100644 --- a/packages/api/src/endpoints/openai/initialize.spec.ts +++ b/packages/api/src/endpoints/openai/initialize.spec.ts @@ -222,6 +222,30 @@ describe('initializeOpenAI – custom headers', () => { jest.clearAllMocks(); }); + it('forwards Agent model transport timeout policy to OpenAI configuration', async () => { + const params = createParams({ OPENAI_API_KEY: 'sk-test' }); + (params.req.config as { endpoints: Record }).endpoints = { + agents: { + modelResponseBodyTimeoutMs: 1_800_000, + modelResponseHeadersTimeoutMs: 120_000, + }, + }; + + try { + await initializeOpenAI(params); + } finally { + (params as unknown as { _restore: () => void })._restore(); + } + + const options = mockGetOpenAIConfig.mock.calls[0][1] as { + transportTimeouts?: { bodyTimeout: number; headersTimeout: number }; + }; + expect(options.transportTimeouts).toEqual({ + bodyTimeout: 1_800_000, + headersTimeout: 120_000, + }); + }); + it('forwards configured endpoint headers (merged over endpoints.all) to getOpenAIConfig', async () => { const params = createParams({ OPENAI_API_KEY: 'sk-test' }); (params.req.config as { endpoints: Record }).endpoints = { diff --git a/packages/api/src/endpoints/openai/initialize.ts b/packages/api/src/endpoints/openai/initialize.ts index 16d412ed34..f7e9acf80d 100644 --- a/packages/api/src/endpoints/openai/initialize.ts +++ b/packages/api/src/endpoints/openai/initialize.ts @@ -12,6 +12,7 @@ import { checkUserKeyExpiry, getAzureCredentials, } from '~/utils'; +import { resolveModelTransportTimeouts } from '~/agents/config'; import { resolveEndpointRuntime } from '~/types'; import { validateEndpointURL } from '~/auth'; import { getOpenAIConfig } from './config'; @@ -94,6 +95,7 @@ export async function initializeOpenAI( reverseProxyUrl: baseURL || undefined, baseURLIsUserProvided: userProvidesURL, allowedAddresses: appConfig?.endpoints?.allowedAddresses, + transportTimeouts: resolveModelTransportTimeouts(appConfig?.endpoints?.agents), streaming: true, }; diff --git a/packages/api/src/endpoints/openai/transport.spec.ts b/packages/api/src/endpoints/openai/transport.spec.ts new file mode 100644 index 0000000000..f0f71abba6 --- /dev/null +++ b/packages/api/src/endpoints/openai/transport.spec.ts @@ -0,0 +1,188 @@ +import { createServer } from 'node:http'; +import { channel } from 'node:diagnostics_channel'; +import { Providers, initializeModel } from '@librechat/agents'; +import type { AddressInfo } from 'node:net'; +import type { Dispatcher } from 'undici'; +import { getOpenAIConfig } from './config'; + +const proxyKeys = [ + 'PROXY', + 'proxy', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'http_proxy', + 'https_proxy', + 'NO_PROXY', + 'no_proxy', +]; +const dispatchers = new Set(); +const originalEnv = process.env; + +beforeEach(() => { + process.env = { ...originalEnv }; + for (const key of proxyKeys) delete process.env[key]; +}); +afterEach(() => { + process.env = originalEnv; +}); +afterAll(async () => { + await Promise.all([...dispatchers].map((dispatcher) => dispatcher.destroy())); +}); + +/** Real sockets and the locked Agent model client: mocked fetch cannot prove Undici's timers. */ +async function request({ + directEndpoint, + mode, + bodyTimeout = 1000, + headersTimeout = 1000, + streaming = true, + cancel = false, + baseURLIsUserProvided = false, +}: { + directEndpoint: boolean; + mode: 'headers' | 'idle' | 'active' | 'redirect'; + bodyTimeout?: number; + headersTimeout?: number; + streaming?: boolean; + cancel?: boolean; + baseURLIsUserProvided?: boolean; +}) { + const transportErrors: string[] = []; + const paths: string[] = []; + const errors = channel('undici:request:error'); + const listener = (event: unknown) => { + const code = (event as { error?: { code?: unknown } }).error?.code; + if (typeof code === 'string' && code.startsWith('UND_ERR_')) transportErrors.push(code); + }; + const server = createServer((req, res) => { + paths.push(req.url ?? ''); + req.resume(); + if (mode === 'redirect') { + res.writeHead(302, { Location: '/must-not-follow' }); + res.end(); + return; + } + const headers = { 'Content-Type': streaming ? 'text/event-stream' : 'application/json' }; + if (mode !== 'headers') { + res.writeHead(200, headers); + res.flushHeaders(); + } + const pulse = + mode === 'active' ? setInterval(() => res.write(': keepalive\n\n'), 100) : undefined; + const finish = setTimeout(() => { + if (mode === 'headers') res.writeHead(200, headers); + res.end( + streaming + ? 'data: [DONE]\n\n' + : JSON.stringify({ + id: 'local', + object: 'chat.completion', + created: 1, + model: 'local-test', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'done' }, + finish_reason: 'stop', + }, + ], + }), + ); + }, 3000); + res.on('close', () => { + clearTimeout(finish); + clearInterval(pulse); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const baseURL = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const controller = new AbortController(); + let timer: ReturnType | undefined; + let failure: unknown; + errors.subscribe(listener); + try { + const { llmConfig, configOptions } = getOpenAIConfig('local-test-only', { + reverseProxyUrl: `${baseURL}${directEndpoint ? '/exact?route=test' : '/v1'}`, + directEndpoint, + baseURLIsUserProvided, + allowedAddresses: baseURLIsUserProvided ? ['127.0.0.1'] : undefined, + transportTimeouts: { bodyTimeout, headersTimeout }, + streaming, + modelOptions: { model: 'local-test' }, + addParams: { timeout: 10_000, maxRetries: 0 }, + }); + dispatchers.add(configOptions!.fetchOptions!.dispatcher as Dispatcher); + const model = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { ...llmConfig, verbosity: undefined, configuration: configOptions }, + }); + if (cancel) timer = setTimeout(() => controller.abort(), 100); + if (streaming) { + const stream = await model.stream('test', { signal: controller.signal }); + for await (const chunk of stream) { + void chunk; + } + } else { + await model.invoke('test', { signal: controller.signal }); + } + } catch (error) { + failure = error; + } finally { + clearTimeout(timer); + errors.unsubscribe(listener); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + return { failure, transportErrors, paths }; +} + +describe.each([false, true])('model transport directEndpoint=%s', (directEndpoint) => { + it('enforces the configured header timeout through the Agent SDK', async () => { + const result = await request({ directEndpoint, mode: 'headers' }); + expect(result.failure).toBeDefined(); + expect(result.transportErrors).toContain('UND_ERR_HEADERS_TIMEOUT'); + expect(result.paths).toEqual([directEndpoint ? '/exact?route=test' : '/v1/chat/completions']); + }); + it('enforces the body-idle timeout after headers', async () => { + const result = await request({ directEndpoint, mode: 'idle' }); + expect(result.failure).toBeDefined(); + expect(result.transportErrors).toContain('UND_ERR_BODY_TIMEOUT'); + expect(result.paths).toHaveLength(1); + }); +}); + +describe('direct endpoint stream lifecycle', () => { + it('allows a longer idle allowance to complete', async () => { + const result = await request({ directEndpoint: true, mode: 'idle', bodyTimeout: 5000 }); + expect(result.failure).toBeUndefined(); + }); + it('resets the idle allowance when data arrives', async () => { + const result = await request({ directEndpoint: true, mode: 'active' }); + expect(result.failure).toBeUndefined(); + }); + it('keeps cancellation independent of disabled transport timers', async () => { + const result = await request({ + directEndpoint: true, + mode: 'idle', + bodyTimeout: 0, + headersTimeout: 0, + cancel: true, + }); + expect(result.failure).toBeDefined(); + expect(result.transportErrors).not.toContain('UND_ERR_BODY_TIMEOUT'); + }); + it('bounds non-streaming response bodies too', async () => { + const result = await request({ directEndpoint: true, mode: 'idle', streaming: false }); + expect(result.failure).toBeDefined(); + expect(result.transportErrors).toContain('UND_ERR_BODY_TIMEOUT'); + }); + it('retains redirect rejection for user-provided URLs', async () => { + const result = await request({ + directEndpoint: true, + mode: 'redirect', + baseURLIsUserProvided: true, + }); + expect(result.failure).toBeDefined(); + expect(result.paths).toEqual(['/exact?route=test']); + }); +}); diff --git a/packages/api/src/types/openai.ts b/packages/api/src/types/openai.ts index 0706a1ccb0..7c876677c1 100644 --- a/packages/api/src/types/openai.ts +++ b/packages/api/src/types/openai.ts @@ -14,6 +14,10 @@ export type OpenAIModelOptions = Partial; */ export interface OpenAIConfigOptions { modelOptions?: OpenAIModelOptions; + transportTimeouts?: { + bodyTimeout: number; + headersTimeout: number; + }; directEndpoint?: boolean; reverseProxyUrl?: string | null; baseURLIsUserProvided?: boolean; diff --git a/packages/api/src/utils/proxy.spec.ts b/packages/api/src/utils/proxy.spec.ts index 774dd627e0..0965bafade 100644 --- a/packages/api/src/utils/proxy.spec.ts +++ b/packages/api/src/utils/proxy.spec.ts @@ -1,7 +1,8 @@ import { HttpsProxyAgent } from 'https-proxy-agent'; -import { EnvHttpProxyAgent, ProxyAgent } from 'undici'; +import { Agent, EnvHttpProxyAgent, ProxyAgent } from 'undici'; import { applyAxiosProxyConfig, + getDirectDispatcher, getEnvProxyDispatcher, getHttpsProxyAgent, getProxyDispatcher, @@ -15,10 +16,12 @@ jest.mock('https-proxy-agent', () => ({ })); jest.mock('undici', () => ({ + Agent: jest.fn(), EnvHttpProxyAgent: jest.fn(), ProxyAgent: jest.fn(), })); +const MockAgent = Agent as jest.MockedClass; const MockEnvHttpProxyAgent = EnvHttpProxyAgent as jest.MockedClass; const MockProxyAgent = ProxyAgent as jest.MockedClass; const MockHttpsProxyAgent = HttpsProxyAgent as jest.MockedClass; @@ -43,6 +46,40 @@ describe('proxy helpers', () => { process.env = originalEnv; }); + it('reuses direct dispatchers with the same transport timeout policy', () => { + const options = { bodyTimeout: 0, headersTimeout: 300_000 }; + + const first = getDirectDispatcher(options); + const second = getDirectDispatcher(options); + + expect(second).toBe(first); + expect(MockAgent).toHaveBeenCalledTimes(1); + expect(MockAgent).toHaveBeenCalledWith(options); + }); + + it('keys only supported timeout options and discards unkeyed connection callbacks', () => { + const options = { bodyTimeout: 123_456, headersTimeout: 654_321, connect: jest.fn() }; + const direct = getDirectDispatcher(options); + expect(getDirectDispatcher({ headersTimeout: 654_321, bodyTimeout: 123_456 })).toBe(direct); + expect(MockAgent).toHaveBeenLastCalledWith({ bodyTimeout: 123_456, headersTimeout: 654_321 }); + + getProxyDispatcher('http://sanitized-proxy:8080', options); + expect(MockProxyAgent).toHaveBeenLastCalledWith({ + uri: 'http://sanitized-proxy:8080', + bodyTimeout: 123_456, + headersTimeout: 654_321, + }); + process.env.PROXY = 'http://sanitized-env-proxy:8080'; + getEnvProxyDispatcher(options); + expect(MockEnvHttpProxyAgent).toHaveBeenLastCalledWith({ + httpProxy: 'http://sanitized-env-proxy:8080', + httpsProxy: 'http://sanitized-env-proxy:8080', + noProxy: undefined, + bodyTimeout: 123_456, + headersTimeout: 654_321, + }); + }); + it('returns undefined when no proxy env is configured', () => { expect(getProxyEnvConfig()).toBeUndefined(); expect(getEnvProxyDispatcher()).toBeUndefined(); @@ -107,6 +144,28 @@ describe('proxy helpers', () => { expect(MockProxyAgent).toHaveBeenCalledWith('http://explicit-proxy:8080'); }); + it('keys proxy dispatchers by transport options', () => { + const proxy = 'http://timeout-proxy:8080'; + + const defaultDispatcher = getProxyDispatcher(proxy); + const longStreamDispatcher = getProxyDispatcher(proxy, { + bodyTimeout: 0, + headersTimeout: 300_000, + }); + const repeatedLongStreamDispatcher = getProxyDispatcher(proxy, { + bodyTimeout: 0, + headersTimeout: 300_000, + }); + + expect(longStreamDispatcher).not.toBe(defaultDispatcher); + expect(repeatedLongStreamDispatcher).toBe(longStreamDispatcher); + expect(MockProxyAgent).toHaveBeenLastCalledWith({ + uri: proxy, + bodyTimeout: 0, + headersTimeout: 300_000, + }); + }); + it('uses the env dispatcher when the explicit proxy matches PROXY', () => { process.env.PROXY = 'http://matching-proxy:8080'; diff --git a/packages/api/src/utils/proxy.ts b/packages/api/src/utils/proxy.ts index bb227ee942..d9b556aa58 100644 --- a/packages/api/src/utils/proxy.ts +++ b/packages/api/src/utils/proxy.ts @@ -1,5 +1,5 @@ import { HttpsProxyAgent } from 'https-proxy-agent'; -import { EnvHttpProxyAgent, ProxyAgent } from 'undici'; +import { Agent, EnvHttpProxyAgent, ProxyAgent } from 'undici'; import type { AxiosRequestConfig, AxiosProxyConfig } from 'axios'; import type { Dispatcher } from 'undici'; @@ -15,8 +15,8 @@ type ProxyResolution = { bypassed: boolean; }; -let envProxyDispatcher: EnvHttpProxyAgent | undefined; -let envProxyDispatcherKey: string | undefined; +const directDispatchers = new Map(); +const envProxyDispatchers = new Map(); const explicitDispatchers = new Map(); const httpsProxyAgents = new Map(); @@ -47,38 +47,83 @@ function getProxyConfigKey(config: ProxyEnvConfig): string { return [config.httpProxy ?? '', config.httpsProxy ?? '', config.noProxy ?? ''].join('|'); } -export function getEnvProxyDispatcher(): Dispatcher | undefined { - const proxyConfig = getProxyEnvConfig(); - if (!proxyConfig) return undefined; - - const key = getProxyConfigKey(proxyConfig); - if (!envProxyDispatcher || envProxyDispatcherKey !== key) { - envProxyDispatcher = new EnvHttpProxyAgent(proxyConfig); - envProxyDispatcherKey = key; - } - - return envProxyDispatcher; +export interface DispatcherTimeoutOptions { + bodyTimeout?: number; + headersTimeout?: number; } -function getExplicitProxyDispatcher(proxyUrl: string): Dispatcher { - const cached = explicitDispatchers.get(proxyUrl); +/** Discard extra runtime properties too: TypeScript cannot prevent structurally wider callers. */ +function pickDispatcherTimeouts({ + bodyTimeout, + headersTimeout, +}: DispatcherTimeoutOptions): DispatcherTimeoutOptions { + return { + ...(bodyTimeout != null ? { bodyTimeout } : {}), + ...(headersTimeout != null ? { headersTimeout } : {}), + }; +} + +function getDispatcherOptionsKey(options: DispatcherTimeoutOptions): string { + return `${options.bodyTimeout ?? ''}|${options.headersTimeout ?? ''}`; +} + +export function getDirectDispatcher(options: DispatcherTimeoutOptions = {}): Dispatcher { + options = pickDispatcherTimeouts(options); + const key = getDispatcherOptionsKey(options); + const cached = directDispatchers.get(key); if (cached) return cached; - const dispatcher = new ProxyAgent(proxyUrl); - explicitDispatchers.set(proxyUrl, dispatcher); + const dispatcher = new Agent(options); + directDispatchers.set(key, dispatcher); return dispatcher; } -export function getProxyDispatcher(proxyUrl?: string | null): Dispatcher | undefined { +export function getEnvProxyDispatcher( + options: DispatcherTimeoutOptions = {}, +): Dispatcher | undefined { + options = pickDispatcherTimeouts(options); + const proxyConfig = getProxyEnvConfig(); + if (!proxyConfig) return undefined; + + const key = `${getProxyConfigKey(proxyConfig)}|${getDispatcherOptionsKey(options)}`; + const cached = envProxyDispatchers.get(key); + if (cached) return cached; + + const dispatcher = new EnvHttpProxyAgent({ ...proxyConfig, ...options }); + envProxyDispatchers.set(key, dispatcher); + return dispatcher; +} + +function getExplicitProxyDispatcher( + proxyUrl: string, + options: DispatcherTimeoutOptions, +): Dispatcher { + options = pickDispatcherTimeouts(options); + const key = `${proxyUrl}|${getDispatcherOptionsKey(options)}`; + const cached = explicitDispatchers.get(key); + if (cached) return cached; + + const dispatcher = + Object.keys(options).length === 0 + ? new ProxyAgent(proxyUrl) + : new ProxyAgent({ uri: proxyUrl, ...options }); + explicitDispatchers.set(key, dispatcher); + return dispatcher; +} + +export function getProxyDispatcher( + proxyUrl?: string | null, + options: DispatcherTimeoutOptions = {}, +): Dispatcher | undefined { const trimmedProxy = proxyUrl?.trim(); - if (!trimmedProxy) return getEnvProxyDispatcher(); + if (!trimmedProxy) return getEnvProxyDispatcher(options); const proxyConfig = getProxyEnvConfig(); if (proxyConfig?.httpProxy === trimmedProxy && proxyConfig?.httpsProxy === trimmedProxy) { - return getEnvProxyDispatcher(); + return getEnvProxyDispatcher(options); } - return getExplicitProxyDispatcher(trimmedProxy); + return getExplicitProxyDispatcher(trimmedProxy, options); } function parseUrl(value: string | URL | undefined): URL | undefined { diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 71e881c9a9..c33514fd2c 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -25,6 +25,35 @@ const endpointsConfig: TEndpointsConfig = { Gemini: { type: EModelEndpoint.custom, userProvide: false, order: 9999 }, }; +describe('agent model response timeouts', () => { + it('ships finite defaults and accepts explicit overrides including disabled timeouts', () => { + expect(agentsEndpointSchema.parse({})).toMatchObject({ + modelResponseBodyTimeoutMs: 900_000, + modelResponseHeadersTimeoutMs: 300_000, + }); + expect( + agentsEndpointSchema.parse({ + modelResponseBodyTimeoutMs: 1_800_000, + modelResponseHeadersTimeoutMs: 0, + }), + ).toMatchObject({ + modelResponseBodyTimeoutMs: 1_800_000, + modelResponseHeadersTimeoutMs: 0, + }); + }); + + it('rejects negative, fractional, and over-one-day values', () => { + for (const value of [-1, 1.5, 86_400_001]) { + expect(agentsEndpointSchema.safeParse({ modelResponseBodyTimeoutMs: value }).success).toBe( + false, + ); + expect(agentsEndpointSchema.safeParse({ modelResponseHeadersTimeoutMs: value }).success).toBe( + false, + ); + } + }); +}); + describe('repository instruction configuration', () => { it('defaults optional reads to two seconds and bounds operator overrides', () => { expect(agentsEndpointSchema.parse({}).repositoryInstructions).toBeUndefined(); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 969b330513..71fff4c12c 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1231,6 +1231,8 @@ export type CodeWorkerEnrollmentPolicy = NonNullable< >; export const DEFAULT_MAX_PROVIDER_ERROR_CHARS = 2000; +export const DEFAULT_AGENT_MODEL_RESPONSE_BODY_TIMEOUT_MS = 900_000; +export const DEFAULT_AGENT_MODEL_RESPONSE_HEADERS_TIMEOUT_MS = 300_000; export const agentsEndpointSchema = baseEndpointSchema .omit({ baseURL: true }) @@ -1244,6 +1246,20 @@ export const agentsEndpointSchema = baseEndpointSchema .min(0) .max(1_000_000) .default(DEFAULT_MAX_PROVIDER_ERROR_CHARS), + /** Maximum inactivity between provider response body chunks; 0 disables the idle timeout. */ + modelResponseBodyTimeoutMs: z + .number() + .int() + .min(0) + .max(86_400_000) + .default(DEFAULT_AGENT_MODEL_RESPONSE_BODY_TIMEOUT_MS), + /** Maximum wait for provider response headers; 0 disables the header timeout. */ + modelResponseHeadersTimeoutMs: z + .number() + .int() + .min(0) + .max(86_400_000) + .default(DEFAULT_AGENT_MODEL_RESPONSE_HEADERS_TIMEOUT_MS), recursionLimit: z.number().optional(), disableBuilder: z.boolean().optional().default(false), /** Optional workspace guidance acquisition budget, separate from command execution. */