mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 15:02:02 +00:00
⛽ feat: Negotiate Background Command Timeouts (#16100)
* ⏳ feat: Negotiate BYOM Background Command Timeouts * ⏱️ fix: Preserve Foreground Programmatic Timeout
This commit is contained in:
parent
61d034c20f
commit
13c74c9574
12 changed files with 147 additions and 12 deletions
|
|
@ -84,6 +84,7 @@ describe('getCodeBridgeWorkerStatus', () => {
|
|||
online: true,
|
||||
ready: true,
|
||||
leaseExpiresInMs: 45_000,
|
||||
maxCommandTimeoutMs: 120_000,
|
||||
capabilities: {
|
||||
statefulWorkspace: true,
|
||||
sandboxProfile: 'native-srt',
|
||||
|
|
@ -114,6 +115,7 @@ describe('getCodeBridgeWorkerStatus', () => {
|
|||
status: 'ready',
|
||||
statefulWorkspace: true,
|
||||
leaseExpiresInMs: 45_000,
|
||||
maxCommandTimeoutMs: 120_000,
|
||||
sandboxProfile: 'native-srt',
|
||||
runtimes: ['bash'],
|
||||
operations: ['read_file', 'execute_command'],
|
||||
|
|
@ -172,6 +174,8 @@ describe('getCodeBridgeWorkerStatus', () => {
|
|||
{ online: true, ready: false },
|
||||
{ online: false, ready: false, leaseExpiresInMs: 5_000 },
|
||||
{ online: true, ready: true, leaseExpiresInMs: 60_001 },
|
||||
{ online: true, ready: true, maxCommandTimeoutMs: 0 },
|
||||
{ online: true, ready: true, maxCommandTimeoutMs: 300_001 },
|
||||
{
|
||||
online: true,
|
||||
ready: true,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS,
|
||||
CODE_WORKSPACE_ID_PATTERN,
|
||||
CODE_WORKSPACE_MAX_COUNT,
|
||||
CODE_WORKSPACE_OPERATIONS,
|
||||
|
|
@ -38,6 +39,7 @@ export type CodeBridgeWorkerStatus = {
|
|||
operations?: CodeWorkspaceOperation[];
|
||||
workspaces?: CodeWorkspaceDescriptor[];
|
||||
programmaticLanguages?: ['bash'];
|
||||
maxCommandTimeoutMs?: number;
|
||||
};
|
||||
|
||||
export type CodeBridgeFetch = (
|
||||
|
|
@ -292,6 +294,7 @@ export async function getCodeBridgeWorkerStatus({
|
|||
online?: unknown;
|
||||
ready?: unknown;
|
||||
leaseExpiresInMs?: unknown;
|
||||
maxCommandTimeoutMs?: unknown;
|
||||
capabilities?: {
|
||||
statefulWorkspace?: unknown;
|
||||
sandboxProfile?: unknown;
|
||||
|
|
@ -306,6 +309,12 @@ export async function getCodeBridgeWorkerStatus({
|
|||
Number.isSafeInteger(status.leaseExpiresInMs) &&
|
||||
status.leaseExpiresInMs > 0 &&
|
||||
status.leaseExpiresInMs <= 60_000);
|
||||
const validCommandTimeout =
|
||||
status.maxCommandTimeoutMs == null ||
|
||||
(typeof status.maxCommandTimeoutMs === 'number' &&
|
||||
Number.isSafeInteger(status.maxCommandTimeoutMs) &&
|
||||
status.maxCommandTimeoutMs >= 1 &&
|
||||
status.maxCommandTimeoutMs <= CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS);
|
||||
const validCapabilities =
|
||||
capabilities == null || validStatusString(capabilities.sandboxProfile);
|
||||
const validRuntimes = capabilities == null || validStatusStringArray(capabilities.runtimes);
|
||||
|
|
@ -322,6 +331,7 @@ export async function getCodeBridgeWorkerStatus({
|
|||
(status.online && (status.leaseExpiresInMs == null || capabilities == null)) ||
|
||||
(!status.online && (status.leaseExpiresInMs != null || capabilities != null)) ||
|
||||
!validLease ||
|
||||
!validCommandTimeout ||
|
||||
!validCapabilities ||
|
||||
!validRuntimes ||
|
||||
!validWorkspaceTools ||
|
||||
|
|
@ -366,6 +376,9 @@ export async function getCodeBridgeWorkerStatus({
|
|||
...(typeof status.leaseExpiresInMs !== 'number'
|
||||
? {}
|
||||
: { leaseExpiresInMs: status.leaseExpiresInMs }),
|
||||
...(typeof status.maxCommandTimeoutMs !== 'number'
|
||||
? {}
|
||||
: { maxCommandTimeoutMs: status.maxCommandTimeoutMs }),
|
||||
...(typeof capabilities?.sandboxProfile !== 'string'
|
||||
? {}
|
||||
: { sandboxProfile: capabilities.sandboxProfile }),
|
||||
|
|
|
|||
|
|
@ -351,7 +351,11 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
|
|||
delete process.env.TEST_CODE_CAPABILITY_TOKEN;
|
||||
});
|
||||
|
||||
function workspaceStatus(workspaces: unknown[], statefulWorkspace: boolean = true): Response {
|
||||
function workspaceStatus(
|
||||
workspaces: unknown[],
|
||||
statefulWorkspace: boolean = true,
|
||||
maxCommandTimeoutMs?: number,
|
||||
): Response {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
|
|
@ -359,6 +363,7 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
|
|||
online: true,
|
||||
ready: true,
|
||||
leaseExpiresInMs: 45_000,
|
||||
maxCommandTimeoutMs,
|
||||
capabilities: {
|
||||
statefulWorkspace,
|
||||
sandboxProfile: 'native-srt',
|
||||
|
|
@ -375,10 +380,14 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
|
|||
|
||||
it('binds the exact advertised workspace and its operation ceiling', async () => {
|
||||
jest.spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
workspaceStatus([
|
||||
{ id: 'project-a', name: 'Project A' },
|
||||
{ id: 'docs', operations: ['read_file', 'list_files'] },
|
||||
]),
|
||||
workspaceStatus(
|
||||
[
|
||||
{ id: 'project-a', name: 'Project A' },
|
||||
{ id: 'docs', operations: ['read_file', 'list_files'] },
|
||||
],
|
||||
true,
|
||||
120_000,
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
|
|
@ -397,6 +406,7 @@ describe('resolveCodeExecutionWorkspaceContext', () => {
|
|||
environmentId: 'personal',
|
||||
workspaceId: 'docs',
|
||||
operations: ['read_file', 'list_files'],
|
||||
maxCommandTimeoutMs: 120_000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -178,6 +178,9 @@ export async function resolveCodeExecutionWorkspaceContext({
|
|||
codeWorkspace: {
|
||||
...selection,
|
||||
operations: [...(workspace.operations ?? status.operations)],
|
||||
...(status.maxCommandTimeoutMs == null
|
||||
? {}
|
||||
: { maxCommandTimeoutMs: status.maxCommandTimeoutMs }),
|
||||
...(workspace.instructions ? { instructions: workspace.instructions } : {}),
|
||||
...(workspace.environment ? { environment: workspace.environment } : {}),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -8,8 +8,10 @@ import {
|
|||
createAttachedWorkspaceBashTool,
|
||||
createGitIdentityProgrammaticBashTool,
|
||||
resolveAttachedWorkspaceCommandTimeoutMax,
|
||||
resolveAttachedWorkspaceProgrammaticTimeout,
|
||||
resolveAttachedWorkspaceQueueWaitMs,
|
||||
} from './command';
|
||||
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation';
|
||||
|
||||
describe('programmatic Bash Git identity', () => {
|
||||
test('applies authorship before the SDK sends a programmatic script', async () => {
|
||||
|
|
@ -88,6 +90,22 @@ describe('programmatic Bash Git identity', () => {
|
|||
await once(server, 'close');
|
||||
}
|
||||
});
|
||||
|
||||
test('preserves the foreground timeout while bounding an explicit admin override', () => {
|
||||
expect(resolveAttachedWorkspaceProgrammaticTimeout(undefined, 90_000)).toBe(30_000);
|
||||
expect(
|
||||
resolveAttachedWorkspaceProgrammaticTimeout(
|
||||
{ limits: { maxCommandTimeoutMs: 120_000 } },
|
||||
90_000,
|
||||
),
|
||||
).toBe(90_000);
|
||||
expect(
|
||||
resolveAttachedWorkspaceProgrammaticTimeout(
|
||||
{ limits: { maxCommandTimeoutMs: 60_000 } },
|
||||
90_000,
|
||||
),
|
||||
).toBe(60_000);
|
||||
});
|
||||
});
|
||||
|
||||
function commandResponse(overrides: Record<string, unknown> = {}): Response {
|
||||
|
|
@ -264,6 +282,41 @@ describe('createAttachedWorkspaceBashTool', () => {
|
|||
properties: { timeoutMs: { type: 'integer', minimum: 1, maximum: 120_000 } },
|
||||
});
|
||||
expect(resolveAttachedWorkspaceCommandTimeoutMax()).toBe(30_000);
|
||||
expect(resolveAttachedWorkspaceCommandTimeoutMax(undefined, 90_000)).toBe(90_000);
|
||||
expect(
|
||||
resolveAttachedWorkspaceCommandTimeoutMax(
|
||||
{ limits: { maxCommandTimeoutMs: 120_000 } },
|
||||
90_000,
|
||||
),
|
||||
).toBe(90_000);
|
||||
expect(
|
||||
resolveAttachedWorkspaceCommandTimeoutMax(
|
||||
{ limits: { maxCommandTimeoutMs: 60_000 } },
|
||||
90_000,
|
||||
),
|
||||
).toBe(60_000);
|
||||
});
|
||||
|
||||
test('uses the negotiated ceiling only for omitted detached background timeouts', async () => {
|
||||
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
|
||||
const bashTool = createAttachedWorkspaceBashTool({
|
||||
baseUrl: 'https://code.example.com/v1',
|
||||
authHeaders: () => ({}),
|
||||
workspaceId: 'project-a',
|
||||
maxTimeoutMs: 90_000,
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
await bashTool.invoke(
|
||||
{ command: 'npm test' },
|
||||
{ configurable: { [BACKGROUND_TOOL_INVOCATION_CONFIG_KEY]: true } },
|
||||
);
|
||||
await bashTool.invoke({ command: 'npm test' });
|
||||
|
||||
const backgroundRequest = JSON.parse(String((fetchImpl as jest.Mock).mock.calls[0][1]?.body));
|
||||
const foregroundRequest = JSON.parse(String((fetchImpl as jest.Mock).mock.calls[1][1]?.body));
|
||||
expect(backgroundRequest).toMatchObject({ timeoutMs: 90_000 });
|
||||
expect(foregroundRequest).toMatchObject({ timeoutMs: 30_000 });
|
||||
});
|
||||
|
||||
test('resolves the administrator-configured admission budget', () => {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
WORKSPACE_COMMAND_MAX_TIMEOUT_MS,
|
||||
WORKSPACE_QUEUE_MAX_WAIT_MS,
|
||||
} from './workspace';
|
||||
import { BACKGROUND_TOOL_INVOCATION_CONFIG_KEY } from '~/agents/invocation';
|
||||
|
||||
const DEFAULT_OUTPUT_BYTES = 256 * 1024;
|
||||
|
||||
|
|
@ -74,7 +75,7 @@ function buildAttachedTimeoutSchema(maxTimeoutMs: number): BoundedTimeoutSchema
|
|||
type: 'integer',
|
||||
minimum: 1,
|
||||
maximum: maxTimeoutMs,
|
||||
description: `Optional execution timeout in milliseconds, from 1 through ${maxTimeoutMs}. Defaults to ${defaultTimeoutMs}. Waiting for an available worker does not consume this execution budget.`,
|
||||
description: `Optional execution timeout in milliseconds, from 1 through ${maxTimeoutMs}. Defaults to ${defaultTimeoutMs} for foreground calls and ${maxTimeoutMs} for detached background calls. Waiting for an available worker does not consume this execution budget.`,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -87,11 +88,41 @@ function normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs: number): numb
|
|||
|
||||
export function resolveAttachedWorkspaceCommandTimeoutMax(
|
||||
configSchema?: CodeEnvironmentUserConfigSchema,
|
||||
upstreamMaxTimeoutMs?: number,
|
||||
): number {
|
||||
const configured = configSchema?.limits?.maxCommandTimeoutMs;
|
||||
return configured == null
|
||||
? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS
|
||||
: normalizeAttachedWorkspaceCommandTimeoutMax(configured);
|
||||
const upstream =
|
||||
upstreamMaxTimeoutMs == null
|
||||
? WORKSPACE_COMMAND_MAX_TIMEOUT_MS
|
||||
: normalizeAttachedWorkspaceCommandTimeoutMax(upstreamMaxTimeoutMs);
|
||||
let requested = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS;
|
||||
if (configured != null) {
|
||||
requested = normalizeAttachedWorkspaceCommandTimeoutMax(configured);
|
||||
} else if (upstreamMaxTimeoutMs != null) {
|
||||
requested = upstream;
|
||||
}
|
||||
return Math.min(requested, upstream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Programmatic calls do not currently carry the detached-invocation marker.
|
||||
* Preserve their historical foreground default while still enforcing both an
|
||||
* explicit administrator override and the live upstream ceiling.
|
||||
*/
|
||||
export function resolveAttachedWorkspaceProgrammaticTimeout(
|
||||
configSchema?: CodeEnvironmentUserConfigSchema,
|
||||
upstreamMaxTimeoutMs?: number,
|
||||
): number {
|
||||
const configured = configSchema?.limits?.maxCommandTimeoutMs;
|
||||
const requested =
|
||||
configured == null
|
||||
? WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS
|
||||
: normalizeAttachedWorkspaceCommandTimeoutMax(configured);
|
||||
const upstream =
|
||||
upstreamMaxTimeoutMs == null
|
||||
? WORKSPACE_COMMAND_MAX_TIMEOUT_MS
|
||||
: normalizeAttachedWorkspaceCommandTimeoutMax(upstreamMaxTimeoutMs);
|
||||
return Math.min(requested, upstream);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -207,8 +238,9 @@ export function createContextProgrammaticBashTool(
|
|||
...(attached
|
||||
? {
|
||||
workspaceId: context.codeWorkspace?.workspaceId,
|
||||
runTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax(
|
||||
runTimeoutMs: resolveAttachedWorkspaceProgrammaticTimeout(
|
||||
context.codeEnvironmentConfigSchema,
|
||||
context.codeWorkspace?.maxCommandTimeoutMs,
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
|
|
@ -259,7 +291,7 @@ export function createAttachedWorkspaceBashTool({
|
|||
workspaceId: string;
|
||||
environment?: CodeWorkspaceDescriptor['environment'];
|
||||
gitIdentity?: AgentGitIdentity | null;
|
||||
/** Deployment ceiling already intersected with the protocol hard cap. */
|
||||
/** Effective admin/upstream ceiling already intersected with the protocol hard cap. */
|
||||
maxTimeoutMs?: number;
|
||||
/** Deployment admission budget; omitted keeps the built-in default. */
|
||||
maxQueueWaitMs?: number;
|
||||
|
|
@ -305,7 +337,10 @@ export function createAttachedWorkspaceBashTool({
|
|||
action ??
|
||||
commandWithGitIdentity(commandWithArguments(rawInput.command!, rawInput.args), gitIdentity);
|
||||
const timeoutMs =
|
||||
rawInput.timeoutMs ?? Math.min(WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, effectiveMaxTimeoutMs);
|
||||
rawInput.timeoutMs ??
|
||||
(config?.configurable?.[BACKGROUND_TOOL_INVOCATION_CONFIG_KEY] === true
|
||||
? effectiveMaxTimeoutMs
|
||||
: Math.min(WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, effectiveMaxTimeoutMs));
|
||||
const signal = config?.signal;
|
||||
const trace = {
|
||||
runId: config?.metadata?.run_id,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue