mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-21 17:44:11 +00:00
🧷 fix: Verify Scheduled HITL State Before Pausing (#15284)
Some checks failed
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Sync Helm Chart Tags / Ignore non-main push (push) Waiting to run
Sync Helm Chart Tags / Sync chart tags (push) Waiting to run
Publish `librechat-data-provider` to NPM / pack (push) Has been cancelled
Publish `librechat-data-provider` to NPM / publish-npm (push) Has been cancelled
Some checks failed
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Sync Helm Chart Tags / Ignore non-main push (push) Waiting to run
Sync Helm Chart Tags / Sync chart tags (push) Waiting to run
Publish `librechat-data-provider` to NPM / pack (push) Has been cancelled
Publish `librechat-data-provider` to NPM / publish-npm (push) Has been cancelled
* fix: require durable storage for scheduled HITL * fix: verify scheduled HITL checkpoints before pause * fix: narrow scheduled HITL capability checks * fix: honor ask tool filters in schedule preflight * fix: enforce durability across scheduled HITL resumes * fix: scope scheduled HITL admission to active tools * refactor: move scheduled HITL admission to TypeScript * fix: close scheduled HITL admission gaps * fix: bind scheduled pauses to current checkpoint * fix: cap pending actions at binding deadline * fix: reject expired pending actions * fix: guard approval deadlines atomically
This commit is contained in:
parent
ab2c0d5362
commit
8ba76507ae
32 changed files with 1524 additions and 102 deletions
|
|
@ -1158,6 +1158,9 @@ HELP_AND_FAQ_URL=https://librechat.ai
|
|||
# USE_REDIS_STREAMS=true
|
||||
# Scheduled chats require shared Redis streams in multi-replica deployments.
|
||||
# Set this only when the deployment truly runs one LibreChat process without Redis.
|
||||
# Scheduled agents that can pause for approval or ask_user_question require both
|
||||
# Redis streams for shared action state and a durable shared checkpointer.
|
||||
# MongoDB is currently the built-in durable checkpointer (and the default).
|
||||
# SCHEDULES_SINGLE_PROCESS=true
|
||||
# Emergency global stop for both automatic and manual scheduled runs.
|
||||
# SCHEDULES_DISABLED=true
|
||||
|
|
|
|||
|
|
@ -382,6 +382,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
|
|||
files: req.body.files,
|
||||
isTemporary: req.body.isTemporary,
|
||||
conversationCreatedAt: req.conversationCreatedAt,
|
||||
isScheduledFire: req._isScheduledFire,
|
||||
timezone: req.body.timezone,
|
||||
checkpointNamespace,
|
||||
requestBody,
|
||||
|
|
@ -722,6 +723,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
|
|||
expect(res.status).toBe(200);
|
||||
await settled;
|
||||
|
||||
expect(capturedInit.isScheduledFire).toBe(true);
|
||||
|
||||
expect(mockClaimScheduleResume).toHaveBeenCalledWith('schedule-1', scheduledFor, {
|
||||
expectedConfigRevision: 4,
|
||||
automatic: true,
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ const {
|
|||
createSubagentUsageSink,
|
||||
anyAgentReplaysReasoningContent,
|
||||
GenerationJobManager,
|
||||
PENDING_ACTION_EXPIRED_CODE,
|
||||
getTransactionsConfig,
|
||||
resolveRecursionLimit,
|
||||
buildPendingAction,
|
||||
|
|
@ -42,13 +43,18 @@ const {
|
|||
pickResumeContext,
|
||||
getApprovalTtlMs,
|
||||
getAgentCheckpointer,
|
||||
hasDurableAgentInterruptCheckpoint,
|
||||
isHITLEnabled,
|
||||
buildToolApprovalHooks,
|
||||
agentRunUsesCheckpointer,
|
||||
canAgentGraphPause,
|
||||
getPluginHookSource,
|
||||
captureAgentCheckpointGeneration,
|
||||
isContentFilterError,
|
||||
deleteAgentCheckpoint,
|
||||
LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
|
||||
LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY,
|
||||
agentRequestsAskUserQuestion,
|
||||
isAskUserQuestionAdminDisabled,
|
||||
attachAskUserQuestionArgs,
|
||||
hydrateResumeRunSteps,
|
||||
createContentIndexOffsetHandlers,
|
||||
|
|
@ -154,19 +160,6 @@ const db = require('~/models');
|
|||
|
||||
const loadAgent = (params) => loadAgentFn(params, { getAgent: db.getAgent, getMCPServerTools });
|
||||
|
||||
function getInterruptTtlMs(checkpointerCfg, req) {
|
||||
const configuredTtlMs = getApprovalTtlMs(checkpointerCfg);
|
||||
const retention = req?._agentEventBindingRetention;
|
||||
if (retention?.expiredAt == null) {
|
||||
return configuredTtlMs;
|
||||
}
|
||||
const bindingDeadline = new Date(retention.expiredAt).getTime();
|
||||
if (!Number.isFinite(bindingDeadline)) {
|
||||
return configuredTtlMs;
|
||||
}
|
||||
return Math.min(configuredTtlMs, Math.max(0, bindingDeadline - Date.now()));
|
||||
}
|
||||
|
||||
const MEMORY_INPUT_CHARS_PER_TOKEN = 8;
|
||||
|
||||
function getUserFacingRequestError(baseMessage, error, appConfig) {
|
||||
|
|
@ -3130,6 +3123,43 @@ class AgentClient extends BaseClient {
|
|||
|
||||
const appConfig = this.options.req?.config;
|
||||
const checkpointerCfg = appConfig?.endpoints?.[EModelEndpoint.agents]?.checkpointer;
|
||||
if (this.options.req?._isScheduledFire === true) {
|
||||
if (!GenerationJobManager.isRedis) {
|
||||
const error = new Error(
|
||||
'The agent paused, but its shared action state is unavailable. Please retry the run.',
|
||||
);
|
||||
error.code = 'SCHEDULED_HITL_REQUIRES_SHARED_STORE';
|
||||
throw error;
|
||||
}
|
||||
let hasDurableInterrupt = false;
|
||||
try {
|
||||
hasDurableInterrupt = await hasDurableAgentInterruptCheckpoint(
|
||||
this.conversationId,
|
||||
checkpointerCfg,
|
||||
{
|
||||
checkpointNamespace: this.checkpointNamespace,
|
||||
checkpointId: interrupt.checkpointId,
|
||||
checkpointNs: interrupt.checkpointNs,
|
||||
interruptId: interrupt.interruptId,
|
||||
},
|
||||
);
|
||||
} catch (checkpointError) {
|
||||
logger.error(
|
||||
`[AgentClient] Failed to verify scheduled HITL checkpoint for ${this.conversationId} (${this.checkpointNamespace || 'legacy namespace'})`,
|
||||
checkpointError,
|
||||
);
|
||||
}
|
||||
if (!hasDurableInterrupt) {
|
||||
logger.error(
|
||||
`[AgentClient] Refusing unresumable scheduled HITL pause for ${this.conversationId} (${this.checkpointNamespace || 'legacy namespace'})`,
|
||||
);
|
||||
const error = new Error(
|
||||
'The agent paused, but its durable continuation checkpoint is unavailable. Please retry the run.',
|
||||
);
|
||||
error.code = 'HITL_CHECKPOINT_UNAVAILABLE';
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
// Persist the generation params (temperature, max tokens, custom endpoint params, …)
|
||||
// so an ephemeral-agent resume continues with the SAME settings the run paused on.
|
||||
// The resume payload omits them and they aren't part of the fingerprint, so without
|
||||
|
|
@ -3174,7 +3204,8 @@ class AgentClient extends BaseClient {
|
|||
// thread_id was bound to conversationId at run config (config.configurable);
|
||||
// fall back to it when the SDK doesn't echo threadId on the interrupt.
|
||||
threadId: interrupt.threadId ?? this.conversationId,
|
||||
ttlMs: getInterruptTtlMs(checkpointerCfg, this.options.req),
|
||||
ttlMs: getApprovalTtlMs(checkpointerCfg),
|
||||
expiresAt: this.options.req?._agentEventBindingRetention?.expiredAt,
|
||||
// Pin the graph-determining request fields so resume can't rebuild this paused
|
||||
// run on a different agent/tool set (esp. ephemeral agents, whose agent_id is
|
||||
// undefined so the id guard can't tell two configs apart).
|
||||
|
|
@ -3281,6 +3312,57 @@ class AgentClient extends BaseClient {
|
|||
abortController = new AbortController();
|
||||
}
|
||||
|
||||
/** Scheduled approvals are unattended by definition. Their pending action,
|
||||
* replay context, and resolution fence must be shared across workers; their
|
||||
* LangGraph continuation must also use a durable shared checkpointer. Redis
|
||||
* provides the first half, while handleRunInterrupt verifies the exact durable
|
||||
* checkpoint before exposing the pause. Refuse unsupported topologies before
|
||||
* spending on provider work. */
|
||||
/** @type {AppConfig['endpoints']['agents']} */
|
||||
const agentsEConfig = appConfig.endpoints?.[EModelEndpoint.agents];
|
||||
const resolvedToolApprovalHooks = isHITLEnabled(agentsEConfig?.toolApproval)
|
||||
? buildToolApprovalHooks({
|
||||
userId: this.options.req?.user?.id,
|
||||
conversationId: this.conversationId,
|
||||
tenantId: this.options.req?.user?.tenantId,
|
||||
appConfig,
|
||||
})
|
||||
: undefined;
|
||||
const topLevelAgents = [this.options.agent, ...(this.agentConfigs?.values() ?? [])];
|
||||
const askUserQuestionAdminDisabled = isAskUserQuestionAdminDisabled(appConfig);
|
||||
const runCanPause = canAgentGraphPause({
|
||||
policy: agentsEConfig?.toolApproval,
|
||||
agents: topLevelAgents,
|
||||
hostGeneratedToolNames:
|
||||
this.options.subagentTasks == null ? undefined : [Constants.CHECK_BACKGROUND_TASK],
|
||||
resolvedProgrammaticHooks: resolvedToolApprovalHooks,
|
||||
pluginHookSource: getPluginHookSource(),
|
||||
askUserQuestionAdminDisabled,
|
||||
});
|
||||
const runUsesCheckpointer = agentRunUsesCheckpointer({
|
||||
policy: agentsEConfig?.toolApproval,
|
||||
agents: topLevelAgents,
|
||||
askUserQuestionAdminDisabled,
|
||||
});
|
||||
if (this.options.req?._isScheduledFire === true && runCanPause) {
|
||||
if (!GenerationJobManager.isRedis) {
|
||||
const error = new Error(
|
||||
'Scheduled agent runs that can pause require a shared generation store. ' +
|
||||
'Enable Redis streams with USE_REDIS_STREAMS=true.',
|
||||
);
|
||||
error.code = 'SCHEDULED_HITL_REQUIRES_SHARED_STORE';
|
||||
throw error;
|
||||
}
|
||||
if (!(await getAgentCheckpointer(agentsEConfig?.checkpointer))) {
|
||||
const error = new Error(
|
||||
'Scheduled agent runs that can pause require a durable shared checkpointer. ' +
|
||||
'Use the default MongoDB checkpointer.',
|
||||
);
|
||||
error.code = 'SCHEDULED_HITL_REQUIRES_DURABLE_CHECKPOINT';
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Fire-and-forget: boot each selected stateful environment in
|
||||
* parallel with generation so the first execute_code/bash call lands
|
||||
* on a warm VM. No-op unless a reachable agent resolved
|
||||
|
|
@ -3291,9 +3373,6 @@ class AgentClient extends BaseClient {
|
|||
agents: [this.options.agent, ...(this.agentConfigs?.values() ?? [])],
|
||||
});
|
||||
|
||||
/** @type {AppConfig['endpoints']['agents']} */
|
||||
const agentsEConfig = appConfig.endpoints?.[EModelEndpoint.agents];
|
||||
|
||||
config = {
|
||||
runName: 'AgentRun',
|
||||
configurable: {
|
||||
|
|
@ -3555,16 +3634,14 @@ class AgentClient extends BaseClient {
|
|||
// HITL is off or the generation has no remnants. Deliberately unconditional
|
||||
// per HITL turn: any cheaper Redis flag can go stale across replicas/restarts,
|
||||
// while these are two indexed, usually-empty deleteMany operations.
|
||||
// The gate mirrors createRun's checkpointer condition: the approval policy
|
||||
// OR an ask_user_question-capable agent (which attaches a checkpointer
|
||||
// WITHOUT the approval policy).
|
||||
// Mirror createRun's checkpointer attachment gate. This is deliberately
|
||||
// broader than pause admission: retries must prune remnants even when a
|
||||
// policy or request hook changed from pausing to non-pausing.
|
||||
//
|
||||
// Start the prune alongside graph construction. The all-settled barrier
|
||||
// below still guarantees it completes before the graph is exposed or run.
|
||||
const shouldPruneCheckpoint =
|
||||
streamId &&
|
||||
this.eventActorInvocationId == null &&
|
||||
(isHITLEnabled(agentsEConfig?.toolApproval) || agents.some(agentRequestsAskUserQuestion));
|
||||
streamId && this.eventActorInvocationId == null && runUsesCheckpointer;
|
||||
let checkpointPrunePromise = Promise.resolve();
|
||||
if (shouldPruneCheckpoint && this.checkpointNamespace !== '') {
|
||||
checkpointPrunePromise = deleteAgentCheckpoint(
|
||||
|
|
@ -3627,6 +3704,7 @@ class AgentClient extends BaseClient {
|
|||
// opts into the tool-approval wiring. Non-resumable callers (OpenAI-compat, Responses)
|
||||
// leave this off so an approval-gated tool can't pause where there's no resume path.
|
||||
hitlCapable: true,
|
||||
resolvedToolApprovalHooks,
|
||||
toolInputValidationErrors: this.toolInputValidationErrors,
|
||||
// Mid-run steering: drain queued user messages at each tool-batch
|
||||
// boundary and inject them into graph state. The offset wrapper
|
||||
|
|
@ -3792,6 +3870,15 @@ class AgentClient extends BaseClient {
|
|||
this.applyHideSequentialOutputsFilter();
|
||||
this.rebaseActivityPhaseBounds(contentBeforeReshape);
|
||||
} catch (err) {
|
||||
if (
|
||||
err?.code === 'SCHEDULED_HITL_REQUIRES_SHARED_STORE' ||
|
||||
err?.code === 'SCHEDULED_HITL_REQUIRES_DURABLE_CHECKPOINT' ||
|
||||
err?.code === 'HITL_CHECKPOINT_UNAVAILABLE' ||
|
||||
err?.code === PENDING_ACTION_EXPIRED_CODE
|
||||
) {
|
||||
logger.warn(`[api/server/controllers/agents/client.js #sendCompletion] ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
if (isContentFilterError(err)) {
|
||||
logger.warn(
|
||||
'[api/server/controllers/agents/client.js #sendCompletion] Blocked by content policy',
|
||||
|
|
@ -3953,6 +4040,14 @@ class AgentClient extends BaseClient {
|
|||
|
||||
/** @type {AppConfig['endpoints']['agents']} */
|
||||
const agentsEConfig = appConfig.endpoints?.[EModelEndpoint.agents];
|
||||
const resolvedToolApprovalHooks = isHITLEnabled(agentsEConfig?.toolApproval)
|
||||
? buildToolApprovalHooks({
|
||||
userId: this.options.req?.user?.id,
|
||||
conversationId: this.conversationId,
|
||||
tenantId: this.options.req?.user?.tenantId,
|
||||
appConfig,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
BaseClient.prototype.setModelBoundStoredMessages.call(
|
||||
this,
|
||||
|
|
@ -4109,6 +4204,7 @@ class AgentClient extends BaseClient {
|
|||
// The resumed run can pause AGAIN (another tool, a follow-up question), and this
|
||||
// controller owns that lifecycle, so it must keep the HITL wiring on the rebuilt run.
|
||||
hitlCapable: true,
|
||||
resolvedToolApprovalHooks,
|
||||
// Plugin SessionStart hooks match on the lifecycle source; a rebuilt run is a
|
||||
// resume, not a fresh startup.
|
||||
sessionStartSource: 'resume',
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ const mockRecordCollectedUsage = jest.fn();
|
|||
const mockDetachedUsageRecorder = jest.fn();
|
||||
const mockCreateDetachedSubagentUsageRecorder = jest.fn(() => mockDetachedUsageRecorder);
|
||||
const mockGetAgentCheckpointer = jest.fn();
|
||||
const mockHasDurableAgentInterruptCheckpoint = jest.fn().mockResolvedValue(true);
|
||||
const mockBuildAgentScopedContext = jest.fn((...args) =>
|
||||
jest.requireActual('@librechat/api').buildAgentScopedContext(...args),
|
||||
);
|
||||
|
|
@ -18,7 +19,14 @@ const mockFormatAgentMessages = jest.fn(() => ({
|
|||
|
||||
const { Providers } = require('@librechat/agents');
|
||||
const { Constants, ContentTypes, EModelEndpoint } = require('librechat-data-provider');
|
||||
const { GenerationJobManager, createStreamServices } = require('@librechat/api');
|
||||
const {
|
||||
GenerationJobManager,
|
||||
createStreamServices,
|
||||
registerToolApprovalHook,
|
||||
clearToolApprovalHooks,
|
||||
getPluginHookSource,
|
||||
setPluginHookSource,
|
||||
} = require('@librechat/api');
|
||||
const BaseClient = require('~/app/clients/BaseClient');
|
||||
const AgentClient = require('./client');
|
||||
const { resolveConfigServers } = require('~/server/services/MCP');
|
||||
|
|
@ -66,6 +74,7 @@ jest.mock('@librechat/api', () => ({
|
|||
maybePrewarmCodeSandbox: jest.fn(),
|
||||
recordCollectedUsage: (...args) => mockRecordCollectedUsage(...args),
|
||||
getAgentCheckpointer: mockGetAgentCheckpointer,
|
||||
hasDurableAgentInterruptCheckpoint: (...args) => mockHasDurableAgentInterruptCheckpoint(...args),
|
||||
}));
|
||||
|
||||
describe('AgentClient - final model-bound content protection', () => {
|
||||
|
|
@ -463,6 +472,7 @@ describe('AgentClient - reasoning label accounting', () => {
|
|||
|
||||
describe('AgentClient - interrupt discovery persistence', () => {
|
||||
beforeEach(async () => {
|
||||
mockHasDurableAgentInterruptCheckpoint.mockClear();
|
||||
await GenerationJobManager.destroy();
|
||||
GenerationJobManager.configure({ ...createStreamServices(), cleanupOnComplete: false });
|
||||
GenerationJobManager.initialize();
|
||||
|
|
@ -567,6 +577,157 @@ describe('AgentClient - interrupt discovery persistence', () => {
|
|||
expect(paused?.metadata.pendingAction.expiresAt).toBeGreaterThanOrEqual(now + 4_900);
|
||||
expect(paused?.metadata.pendingAction.expiresAt).toBeLessThanOrEqual(now + 5_000);
|
||||
});
|
||||
|
||||
it('does not expose an event-bound pause after its inherited deadline', async () => {
|
||||
const streamId = 'conversation-expired-event-bound-pause';
|
||||
const job = await GenerationJobManager.createJob(streamId, 'user-123', streamId);
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: { endpoint: EModelEndpoint.agents, agent_id: 'agent-123' },
|
||||
config: { endpoints: { [EModelEndpoint.agents]: { checkpointer: { ttl: 3600 } } } },
|
||||
_agentEventBindingRetention: {
|
||||
isTemporary: false,
|
||||
expiredAt: new Date(Date.now() - 1),
|
||||
},
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = streamId;
|
||||
client.responseMessageId = 'response-expired-event-bound-pause';
|
||||
client.jobCreatedAt = job.createdAt;
|
||||
|
||||
await expect(
|
||||
client.handleRunInterrupt(
|
||||
{
|
||||
getInterrupt: () => ({
|
||||
interruptId: 'ask-interrupt',
|
||||
threadId: streamId,
|
||||
payload: {
|
||||
type: 'ask_user_question',
|
||||
question: { question: 'Proceed?' },
|
||||
},
|
||||
}),
|
||||
},
|
||||
streamId,
|
||||
),
|
||||
).rejects.toMatchObject({ code: 'HITL_ACTION_EXPIRED' });
|
||||
const liveJob = await GenerationJobManager.getJob(streamId);
|
||||
expect(liveJob?.status).toBe('running');
|
||||
expect(liveJob?.metadata.pendingAction).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not expose a scheduled pause without its shared action store', async () => {
|
||||
const streamId = 'scheduled-missing-shared-store';
|
||||
const job = await GenerationJobManager.createJob(streamId, 'user-123', streamId);
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: { endpoint: EModelEndpoint.agents, agent_id: 'agent-123' },
|
||||
config: { endpoints: { [EModelEndpoint.agents]: {} } },
|
||||
_isScheduledFire: true,
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = streamId;
|
||||
client.responseMessageId = 'scheduled-missing-shared-store-response';
|
||||
client.jobCreatedAt = job.createdAt;
|
||||
client.checkpointNamespace = job.metadata.checkpointNamespace;
|
||||
|
||||
await expect(
|
||||
client.handleRunInterrupt(
|
||||
{
|
||||
getInterrupt: () => ({
|
||||
interruptId: 'ask-interrupt',
|
||||
threadId: streamId,
|
||||
payload: {
|
||||
type: 'ask_user_question',
|
||||
question: { question: 'Proceed?' },
|
||||
},
|
||||
}),
|
||||
},
|
||||
streamId,
|
||||
),
|
||||
).rejects.toMatchObject({ code: 'SCHEDULED_HITL_REQUIRES_SHARED_STORE' });
|
||||
expect(mockHasDurableAgentInterruptCheckpoint).not.toHaveBeenCalled();
|
||||
await expect(GenerationJobManager.getJobStatus(streamId)).resolves.toBe('running');
|
||||
});
|
||||
|
||||
it('does not expose a scheduled pause without its durable interrupt checkpoint', async () => {
|
||||
const isRedisSpy = jest.spyOn(GenerationJobManager, 'isRedis', 'get').mockReturnValue(true);
|
||||
const streamId = 'scheduled-missing-interrupt-checkpoint';
|
||||
const job = await GenerationJobManager.createJob(streamId, 'user-123', streamId);
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: { endpoint: EModelEndpoint.agents, agent_id: 'agent-123' },
|
||||
config: { endpoints: { [EModelEndpoint.agents]: {} } },
|
||||
_isScheduledFire: true,
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = streamId;
|
||||
client.responseMessageId = 'scheduled-missing-checkpoint-response';
|
||||
client.jobCreatedAt = job.createdAt;
|
||||
client.checkpointNamespace = job.metadata.checkpointNamespace;
|
||||
mockHasDurableAgentInterruptCheckpoint.mockResolvedValueOnce(false);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
client.handleRunInterrupt(
|
||||
{
|
||||
getInterrupt: () => ({
|
||||
interruptId: 'ask-interrupt',
|
||||
checkpointId: 'checkpoint-current',
|
||||
checkpointNs: 'nested-agent',
|
||||
threadId: streamId,
|
||||
payload: {
|
||||
type: 'ask_user_question',
|
||||
question: { question: 'Proceed?' },
|
||||
},
|
||||
}),
|
||||
},
|
||||
streamId,
|
||||
),
|
||||
).rejects.toMatchObject({ code: 'HITL_CHECKPOINT_UNAVAILABLE' });
|
||||
expect(mockHasDurableAgentInterruptCheckpoint).toHaveBeenCalledWith(streamId, undefined, {
|
||||
checkpointNamespace: job.metadata.checkpointNamespace,
|
||||
checkpointId: 'checkpoint-current',
|
||||
checkpointNs: 'nested-agent',
|
||||
interruptId: 'ask-interrupt',
|
||||
});
|
||||
await expect(GenerationJobManager.getJobStatus(streamId)).resolves.toBe('running');
|
||||
} finally {
|
||||
isRedisSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
|
|
@ -817,9 +978,280 @@ describe('AgentClient - activity phase completion', () => {
|
|||
|
||||
describe('AgentClient - startup telemetry', () => {
|
||||
afterEach(() => {
|
||||
clearToolApprovalHooks();
|
||||
mockIsHITLEnabled.mockReturnValue(false);
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('refuses scheduled pause-capable runs before provider startup without Redis', async () => {
|
||||
mockIsHITLEnabled.mockReturnValue(false);
|
||||
const createRunBefore = mockCreateRun.mock.calls.length;
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: { endpoints: { [EModelEndpoint.agents]: {} } },
|
||||
_isScheduledFire: true,
|
||||
_resumableStreamId: 'scheduled-hitl-no-redis',
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
tools: [{ name: 'ask_user_question' }],
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = 'scheduled-hitl-no-redis';
|
||||
client.responseMessageId = 'scheduled-hitl-response';
|
||||
client.parentMessageId = 'scheduled-hitl-parent';
|
||||
|
||||
await expect(client.chatCompletion({ payload: [] })).rejects.toMatchObject({
|
||||
code: 'SCHEDULED_HITL_REQUIRES_SHARED_STORE',
|
||||
message: expect.stringContaining('USE_REDIS_STREAMS=true'),
|
||||
});
|
||||
expect(mockCreateRun).toHaveBeenCalledTimes(createRunBefore);
|
||||
});
|
||||
|
||||
it('includes host-generated background tools in scheduled pause admission', async () => {
|
||||
const createRunBefore = mockCreateRun.mock.calls.length;
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: {
|
||||
endpoints: {
|
||||
[EModelEndpoint.agents]: {
|
||||
toolApproval: {
|
||||
enabled: true,
|
||||
mode: 'bypass',
|
||||
ask: ['check_background_task'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
_isScheduledFire: true,
|
||||
_resumableStreamId: 'scheduled-background-tool',
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
tools: [{ name: 'read_file' }],
|
||||
},
|
||||
subagentTasks: {},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = 'scheduled-background-tool';
|
||||
client.responseMessageId = 'scheduled-background-response';
|
||||
client.parentMessageId = 'scheduled-background-parent';
|
||||
|
||||
await expect(client.chatCompletion({ payload: [] })).rejects.toMatchObject({
|
||||
code: 'SCHEDULED_HITL_REQUIRES_SHARED_STORE',
|
||||
});
|
||||
expect(mockCreateRun).toHaveBeenCalledTimes(createRunBefore);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: 'a bypass-only approval policy',
|
||||
toolApproval: { enabled: true, mode: 'bypass' },
|
||||
subagentAgentConfigs: undefined,
|
||||
},
|
||||
{
|
||||
name: 'an approval rule that matches no selected tool',
|
||||
toolApproval: { enabled: true, mode: 'bypass', ask: ['approval_probe'] },
|
||||
subagentAgentConfigs: undefined,
|
||||
},
|
||||
{
|
||||
name: 'ask_user_question denied by the approval policy',
|
||||
toolApproval: { enabled: true, deny: ['ask_*'] },
|
||||
primaryTools: [{ name: 'ask_user_question' }],
|
||||
subagentAgentConfigs: undefined,
|
||||
},
|
||||
{
|
||||
name: 'ask_user_question on a nested subagent only',
|
||||
toolApproval: undefined,
|
||||
subagentAgentConfigs: [
|
||||
{
|
||||
id: 'nested-agent',
|
||||
tools: [{ name: 'ask_user_question' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
])(
|
||||
'does not reject scheduled runs for $name',
|
||||
async ({ toolApproval, primaryTools, subagentAgentConfigs }) => {
|
||||
mockDeleteAgentCheckpoint.mockReset().mockResolvedValue(undefined);
|
||||
const processStream = jest.fn().mockResolvedValue();
|
||||
mockCreateRun.mockResolvedValueOnce({
|
||||
Graph: null,
|
||||
processStream,
|
||||
getCalibrationRatio: jest.fn(() => 0),
|
||||
getInterrupt: jest.fn(() => undefined),
|
||||
});
|
||||
const createRunBefore = mockCreateRun.mock.calls.length;
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: { endpoints: { [EModelEndpoint.agents]: { toolApproval } } },
|
||||
_isScheduledFire: true,
|
||||
_resumableStreamId: 'scheduled-non-pausing-policy',
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
hide_sequential_outputs: false,
|
||||
tools: primaryTools ?? [{ name: 'read_file' }],
|
||||
subagentAgentConfigs,
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = 'scheduled-non-pausing-policy';
|
||||
client.checkpointNamespace = 'scheduled-non-pausing-generation';
|
||||
client.responseMessageId = 'scheduled-non-pausing-response';
|
||||
client.parentMessageId = 'scheduled-non-pausing-parent';
|
||||
client.recordCollectedUsage = jest.fn().mockResolvedValue();
|
||||
|
||||
await expect(client.chatCompletion({ payload: [] })).resolves.toBeUndefined();
|
||||
expect(mockCreateRun).toHaveBeenCalledTimes(createRunBefore + 1);
|
||||
expect(processStream).toHaveBeenCalledTimes(1);
|
||||
if (toolApproval?.enabled === true) {
|
||||
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
|
||||
'scheduled-non-pausing-policy',
|
||||
undefined,
|
||||
undefined,
|
||||
{
|
||||
throwOnError: true,
|
||||
checkpointNamespace: 'scheduled-non-pausing-generation',
|
||||
},
|
||||
);
|
||||
} else {
|
||||
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('uses request-scoped hook resolution when deciding whether a scheduled run can pause', async () => {
|
||||
mockIsHITLEnabled.mockReturnValue(true);
|
||||
registerToolApprovalHook((context) =>
|
||||
context.userId === 'different-user' ? async () => ({ decision: 'ask' }) : undefined,
|
||||
);
|
||||
const processStream = jest.fn().mockResolvedValue();
|
||||
mockCreateRun.mockResolvedValueOnce({
|
||||
Graph: null,
|
||||
processStream,
|
||||
getCalibrationRatio: jest.fn(() => 0),
|
||||
getInterrupt: jest.fn(() => undefined),
|
||||
});
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: {
|
||||
endpoints: {
|
||||
[EModelEndpoint.agents]: {
|
||||
toolApproval: { enabled: true, mode: 'bypass' },
|
||||
},
|
||||
},
|
||||
},
|
||||
_isScheduledFire: true,
|
||||
_resumableStreamId: 'scheduled-request-scoped-hook',
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
hide_sequential_outputs: false,
|
||||
tools: [{ name: 'read_file' }],
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = 'scheduled-request-scoped-hook';
|
||||
client.checkpointNamespace = 'scheduled-request-scoped-generation';
|
||||
client.responseMessageId = 'scheduled-request-scoped-response';
|
||||
client.parentMessageId = 'scheduled-request-scoped-parent';
|
||||
client.recordCollectedUsage = jest.fn().mockResolvedValue();
|
||||
|
||||
await expect(client.chatCompletion({ payload: [] })).resolves.toBeUndefined();
|
||||
expect(mockCreateRun.mock.calls.at(-1)?.[0]?.resolvedToolApprovalHooks).toEqual([]);
|
||||
});
|
||||
|
||||
it('admits deployment PreToolUse hooks as pause-capable for scheduled runs', async () => {
|
||||
mockIsHITLEnabled.mockReturnValue(true);
|
||||
const previousSource = getPluginHookSource();
|
||||
setPluginHookSource({
|
||||
hasHooks: () => true,
|
||||
hasToolApprovalHooks: () => true,
|
||||
register: () => 1,
|
||||
});
|
||||
const client = new AgentClient({
|
||||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: {
|
||||
endpoints: {
|
||||
[EModelEndpoint.agents]: {
|
||||
toolApproval: { enabled: true, mode: 'bypass' },
|
||||
},
|
||||
},
|
||||
},
|
||||
_isScheduledFire: true,
|
||||
_resumableStreamId: 'scheduled-deployment-hook',
|
||||
},
|
||||
res: {},
|
||||
agent: {
|
||||
id: 'agent-123',
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
tools: [{ name: 'write_file' }],
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
contentParts: [],
|
||||
collectedUsage: [],
|
||||
artifactPromises: [],
|
||||
});
|
||||
client.conversationId = 'scheduled-deployment-hook';
|
||||
client.responseMessageId = 'scheduled-deployment-response';
|
||||
client.parentMessageId = 'scheduled-deployment-parent';
|
||||
|
||||
try {
|
||||
await expect(client.chatCompletion({ payload: [] })).rejects.toMatchObject({
|
||||
code: 'SCHEDULED_HITL_REQUIRES_SHARED_STORE',
|
||||
});
|
||||
} finally {
|
||||
setPluginHookSource(previousSource);
|
||||
}
|
||||
});
|
||||
|
||||
it('overlaps run creation with checkpoint pruning and joins both before stream processing', async () => {
|
||||
let releaseCheckpoint;
|
||||
let checkpointStarted;
|
||||
|
|
@ -853,7 +1285,9 @@ describe('AgentClient - startup telemetry', () => {
|
|||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: { endpoints: { [EModelEndpoint.agents]: { toolApproval: {} } } },
|
||||
config: {
|
||||
endpoints: { [EModelEndpoint.agents]: { toolApproval: { enabled: true } } },
|
||||
},
|
||||
_resumableStreamId: 'conversation-123',
|
||||
},
|
||||
res: {},
|
||||
|
|
@ -863,6 +1297,7 @@ describe('AgentClient - startup telemetry', () => {
|
|||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
hide_sequential_outputs: false,
|
||||
tools: [{ name: 'write_file' }],
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
|
|
@ -1094,7 +1529,9 @@ describe('AgentClient - startup telemetry', () => {
|
|||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: { endpoints: { [EModelEndpoint.agents]: { toolApproval: {} } } },
|
||||
config: {
|
||||
endpoints: { [EModelEndpoint.agents]: { toolApproval: { enabled: true } } },
|
||||
},
|
||||
_resumableStreamId: 'conversation-123',
|
||||
},
|
||||
res: {},
|
||||
|
|
@ -1104,6 +1541,7 @@ describe('AgentClient - startup telemetry', () => {
|
|||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
hide_sequential_outputs: false,
|
||||
tools: [{ name: 'write_file' }],
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
|
|
@ -1168,7 +1606,9 @@ describe('AgentClient - startup telemetry', () => {
|
|||
req: {
|
||||
user: { id: 'user-123' },
|
||||
body: {},
|
||||
config: { endpoints: { [EModelEndpoint.agents]: { toolApproval: {} } } },
|
||||
config: {
|
||||
endpoints: { [EModelEndpoint.agents]: { toolApproval: { enabled: true } } },
|
||||
},
|
||||
_resumableStreamId: 'conversation-123',
|
||||
},
|
||||
res: {},
|
||||
|
|
@ -1178,6 +1618,7 @@ describe('AgentClient - startup telemetry', () => {
|
|||
provider: EModelEndpoint.openAI,
|
||||
model_parameters: { model: 'gpt-4' },
|
||||
hide_sequential_outputs: false,
|
||||
tools: [{ name: 'write_file' }],
|
||||
},
|
||||
endpointTokenConfig: {},
|
||||
eventHandlers: {},
|
||||
|
|
|
|||
|
|
@ -712,6 +712,13 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
|
|||
|
||||
const scheduleId = job.metadata?.scheduleId;
|
||||
const scheduledFor = job.metadata?.scheduledFor;
|
||||
// A resumed schedule is still an unattended scheduled run. Preserve that
|
||||
// provenance on the rebuilt client so every later pause (a second approval
|
||||
// or follow-up question) must prove its durable checkpoint before the job is
|
||||
// exposed as `requires_action` again.
|
||||
if (scheduleId) {
|
||||
req._isScheduledFire = true;
|
||||
}
|
||||
if (
|
||||
scheduleId &&
|
||||
!(await isScheduleLive(scheduleId, job.metadata?.scheduleConfigRevision, {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ const {
|
|||
getDeploymentPluginHookCapabilities,
|
||||
registerDeploymentPluginHooks,
|
||||
hasDeploymentPluginHooks,
|
||||
hasDeploymentPluginToolApprovalHooks,
|
||||
setPluginHookSource,
|
||||
loadToolApprovalHooks,
|
||||
maybeInjectQueryDevtoolsBootstrap,
|
||||
|
|
@ -220,6 +221,7 @@ const startServer = async () => {
|
|||
// agents -> plugins import (see agents/hooks/source.ts).
|
||||
setPluginHookSource({
|
||||
hasHooks: hasDeploymentPluginHooks,
|
||||
hasToolApprovalHooks: hasDeploymentPluginToolApprovalHooks,
|
||||
register: registerDeploymentPluginHooks,
|
||||
});
|
||||
await initializeDeploymentSkills({
|
||||
|
|
|
|||
|
|
@ -200,6 +200,9 @@ interface:
|
|||
# A deployment with Redis-backed resumable streams can run this on every replica.
|
||||
# A single-process deployment without Redis must also set
|
||||
# `SCHEDULES_SINGLE_PROCESS=true`; unsafe multi-replica writes fail closed.
|
||||
# Scheduled agents that can pause require both `USE_REDIS_STREAMS=true` for
|
||||
# shared action state and a durable shared checkpointer for graph continuation.
|
||||
# MongoDB is currently the built-in durable checkpointer (and the default).
|
||||
# schedules:
|
||||
# use: true
|
||||
# create: true
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import type { BaseMessage } from '@langchain/core/messages';
|
|||
import type { SubagentTaskConfig } from '@librechat/agents';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent';
|
||||
import { createRun } from '~/agents/run';
|
||||
import { createRun, isAskUserQuestionAdminDisabled } from '~/agents/run';
|
||||
|
||||
// Mock winston logger — `format` must be callable so @librechat/data-schemas
|
||||
// dist module-load completes cleanly; see api/test/__mocks__/logger.js.
|
||||
|
|
@ -107,6 +107,21 @@ function makeAgent(
|
|||
};
|
||||
}
|
||||
|
||||
describe('isAskUserQuestionAdminDisabled', () => {
|
||||
it('applies includedTools precedence and the filteredTools fallback', () => {
|
||||
expect(isAskUserQuestionAdminDisabled(undefined)).toBe(false);
|
||||
expect(isAskUserQuestionAdminDisabled({ includedTools: ['calculator'] } as AppConfig)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
isAskUserQuestionAdminDisabled({ includedTools: ['ask_user_question'] } as AppConfig),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isAskUserQuestionAdminDisabled({ filteredTools: ['ask_user_question'] } as AppConfig),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
type TestRunAgent = ReturnType<typeof makeAgent> & {
|
||||
subagentAgentConfigs?: TestRunAgent[];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb';
|
|||
import { emptyCheckpoint, ERROR, INTERRUPT } from '@langchain/langgraph-checkpoint';
|
||||
import {
|
||||
getAgentCheckpointer,
|
||||
hasDurableAgentInterruptCheckpoint,
|
||||
captureAgentCheckpointGeneration,
|
||||
deleteAgentCheckpoint,
|
||||
deleteAgentCheckpoints,
|
||||
|
|
@ -49,6 +50,7 @@ async function seedInterruptCheckpoint(
|
|||
saver: MongoDBSaver,
|
||||
threadId: string,
|
||||
checkpointNamespace = '',
|
||||
interruptId = 'interrupt-current',
|
||||
) {
|
||||
const { config, checkpoint, metadata } = putArgs(threadId, checkpointNamespace);
|
||||
await saver.putWrites(
|
||||
|
|
@ -59,7 +61,7 @@ async function seedInterruptCheckpoint(
|
|||
checkpoint_id: checkpoint.id,
|
||||
},
|
||||
},
|
||||
[[INTERRUPT, 'approve?']],
|
||||
[[INTERRUPT, { id: interruptId, value: 'approve?' }]],
|
||||
'task-1',
|
||||
);
|
||||
await saver.put(config, checkpoint, metadata);
|
||||
|
|
@ -98,6 +100,68 @@ describe('checkpointer (mongodb-memory-server integration)', () => {
|
|||
expect(ttlIndex?.expireAfterSeconds).toBe(3600);
|
||||
});
|
||||
|
||||
it('verifies that an interrupt write has its matching durable checkpoint', async () => {
|
||||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||||
expect(saver).toBeDefined();
|
||||
const missingIdentity = {
|
||||
checkpointId: 'missing-checkpoint',
|
||||
interruptId: 'interrupt-current',
|
||||
};
|
||||
await expect(
|
||||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, missingIdentity),
|
||||
).resolves.toBe(false);
|
||||
|
||||
const checkpoint = await seedInterruptCheckpoint(saver!, 'verified-pause');
|
||||
const identity = {
|
||||
checkpointId: checkpoint.id,
|
||||
interruptId: 'interrupt-current',
|
||||
};
|
||||
|
||||
await expect(
|
||||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, identity),
|
||||
).resolves.toBe(true);
|
||||
await expect(
|
||||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, missingIdentity),
|
||||
).resolves.toBe(false);
|
||||
await expect(
|
||||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, {
|
||||
...identity,
|
||||
interruptId: 'interrupt-stale',
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
await mongoose.connection.db!.collection('agent_checkpoints').deleteMany({
|
||||
thread_id: 'verified-pause',
|
||||
});
|
||||
await expect(
|
||||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, identity),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("uses Mongoose's selected database instead of the MongoClient URI default", async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongoose.connect(mongoServer.getUri('driver_default'), { dbName: 'active_app' });
|
||||
__resetCheckpointerForTests();
|
||||
|
||||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||||
expect(saver).toBeDefined();
|
||||
await seedInterruptCheckpoint(saver!, 'db-selection-pause');
|
||||
|
||||
expect(
|
||||
await mongoose.connection.db!.collection('agent_checkpoints').countDocuments({
|
||||
thread_id: 'db-selection-pause',
|
||||
}),
|
||||
).toBe(1);
|
||||
expect(
|
||||
await mongoose.connection
|
||||
.getClient()
|
||||
.db('driver_default')
|
||||
.collection('agent_checkpoints')
|
||||
.countDocuments({
|
||||
thread_id: 'db-selection-pause',
|
||||
}),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it('returns undefined for the memory type (SDK MemorySaver fallback) even when connected', async () => {
|
||||
expect(await getAgentCheckpointer({ type: 'memory' })).toBeUndefined();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -403,6 +403,9 @@ export class LazyMongoSaver extends MongoDBSaver {
|
|||
if (isEventActorInvocation(config)) {
|
||||
await this.assertCheckpointFitsDocument(config, checkpoint, metadata);
|
||||
const persisted = await super.put(toStorageCheckpointConfig(config), checkpoint, metadata);
|
||||
logger.debug(
|
||||
`[checkpointer] Persisted durable checkpoint for thread ${config.configurable?.thread_id ?? 'unknown'} (${checkpoint.id})`,
|
||||
);
|
||||
return fromStorageCheckpointConfig(persisted, config);
|
||||
}
|
||||
if (this.writeAnchorIds.delete(checkpoint.id)) {
|
||||
|
|
@ -413,6 +416,9 @@ export class LazyMongoSaver extends MongoDBSaver {
|
|||
sweepStale(this.persistedIds, (t) => t);
|
||||
this.persistedIds.set(checkpoint.id, Date.now());
|
||||
const persisted = await super.put(toStorageCheckpointConfig(config), checkpoint, metadata);
|
||||
logger.debug(
|
||||
`[checkpointer] Persisted durable checkpoint for thread ${config.configurable?.thread_id ?? 'unknown'} (${checkpoint.id})`,
|
||||
);
|
||||
// `assertCheckpointFitsDocument` awaits a (potentially slow) serialization AFTER the
|
||||
// anchor was consumed above but BEFORE `persistedIds` was set — a bookkeeping-only
|
||||
// `putWrites` dispatched in that window sees neither marker and parks its batch. Flush
|
||||
|
|
@ -512,7 +518,7 @@ export class LazyMongoSaver extends MongoDBSaver {
|
|||
return;
|
||||
}
|
||||
logger.debug(
|
||||
`[checkpointer] Persisting durable checkpoint for thread ${threadId ?? 'unknown'}: ${bytes} bytes`,
|
||||
`[checkpointer] Prepared durable checkpoint for thread ${threadId ?? 'unknown'}: ${bytes} bytes`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -587,6 +593,62 @@ export function getApprovalTtlMs(cfg: TCheckpointerConfig | undefined): number {
|
|||
return resolveCheckpointerConfig(cfg).ttlSeconds * 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prove that the durable saver contains a complete interrupt checkpoint for one generation.
|
||||
*
|
||||
* A pending Redis action is useful only when LangGraph can reload the state it
|
||||
* interrupted. Read the exact checkpoint selected by the current interrupt and
|
||||
* require its matching interrupt id, so an older retained pause cannot satisfy
|
||||
* verification for a missing or misrouted re-pause.
|
||||
* This runs once per interrupt, never on the ordinary generation path.
|
||||
*/
|
||||
export async function hasDurableAgentInterruptCheckpoint(
|
||||
threadId: string,
|
||||
cfg?: TCheckpointerConfig,
|
||||
options?: {
|
||||
checkpointNamespace?: string;
|
||||
checkpointId: string;
|
||||
checkpointNs?: string;
|
||||
interruptId: string;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
if (!threadId || !options?.checkpointId || !options.interruptId) {
|
||||
return false;
|
||||
}
|
||||
const saver = await getAgentCheckpointer(cfg);
|
||||
if (!saver) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const checkpointNamespace = options?.checkpointNamespace ?? '';
|
||||
const tuple = await saver.getTuple({
|
||||
configurable: {
|
||||
thread_id: threadId,
|
||||
checkpoint_ns: options.checkpointNs ?? '',
|
||||
checkpoint_id: options.checkpointId,
|
||||
...(checkpointNamespace !== '' && {
|
||||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
|
||||
}),
|
||||
},
|
||||
});
|
||||
if (tuple?.checkpoint.id !== options.checkpointId) {
|
||||
return false;
|
||||
}
|
||||
return (tuple.pendingWrites ?? []).some((write) => {
|
||||
if (write[1] !== INTERRUPT) {
|
||||
return false;
|
||||
}
|
||||
const values = Array.isArray(write[2]) ? write[2] : [write[2]];
|
||||
return values.some(
|
||||
(value) =>
|
||||
value != null &&
|
||||
typeof value === 'object' &&
|
||||
'id' in value &&
|
||||
value.id === options.interruptId,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One saver per process, built lazily on first use so `setup()` (index creation)
|
||||
* runs exactly once. Keyed by the resolved settings so a config change rebuilds.
|
||||
|
|
@ -711,6 +773,13 @@ async function buildMongoSaver(
|
|||
client: mongoose.connection.getClient() as unknown as ConstructorParameters<
|
||||
typeof MongoDBSaver
|
||||
>[0]['client'],
|
||||
// MongoDBSaver calls MongoClient.db(dbName). Passing no name makes that
|
||||
// resolve from the driver's URI default, which is not guaranteed to be the
|
||||
// database Mongoose selected (for example when Mongoose connected with a
|
||||
// dbName override). Every capture/delete path below uses connection.db, so
|
||||
// bind the saver to that exact database as well or a pause can be written to
|
||||
// one database while LibreChat looks for it in another.
|
||||
dbName: mongoose.connection.db?.databaseName,
|
||||
checkpointCollectionName: resolved.checkpointCollectionName,
|
||||
checkpointWritesCollectionName: resolved.checkpointWritesCollectionName,
|
||||
// TTL index on `upserted_at`: an unresolved paused run is reclaimed after the
|
||||
|
|
|
|||
163
packages/api/src/agents/hitl/admission.spec.ts
Normal file
163
packages/api/src/agents/hitl/admission.spec.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
import type { PluginHookSource } from '~/agents/hooks/source';
|
||||
import type { ToolApprovalAdmissionAgent } from './admission';
|
||||
import type { ToolApprovalHook } from './hooks';
|
||||
import { agentRunUsesCheckpointer, canAgentGraphPause } from './admission';
|
||||
|
||||
const askHook: ToolApprovalHook = async () => ({ decision: 'ask' });
|
||||
|
||||
function pluginSource(
|
||||
hasToolApprovalHooks: PluginHookSource['hasToolApprovalHooks'],
|
||||
): PluginHookSource {
|
||||
return {
|
||||
hasHooks: () => true,
|
||||
hasToolApprovalHooks,
|
||||
register: () => 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe('canAgentGraphPause', () => {
|
||||
test.each([
|
||||
['configured tool names', { tools: ['read_file'] }, 'read_file'],
|
||||
['loaded tool objects', { tools: [{ name: 'read_file' }] }, 'read_file'],
|
||||
['tool definitions', { toolDefinitions: [{ name: 'read_file' }] }, 'read_file'],
|
||||
['tool registries', { toolRegistry: new Map([['read_file', {}]]) }, 'read_file'],
|
||||
])('discovers %s', (_label, agent, toolName) => {
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass', ask: [toolName] },
|
||||
agents: [agent],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['initialized children', { subagentAgentConfigs: [{ tools: ['write_file'] }] }],
|
||||
['lazy children', { lazySubagentConfigs: [{ tools: ['write_file'] }] }],
|
||||
['graph members', { subagentGraphConfigs: [{ memberConfigs: [{ tools: ['write_file'] }] }] }],
|
||||
['graph member metadata', { subagentGraphMemberMetadata: [{ tools: ['write_file'] }] }],
|
||||
])('intersects approval policy with %s', (_label, agent) => {
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass', ask: ['write_*'] },
|
||||
agents: [agent],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test('fails closed for an unresolved lazy tool surface that could pause', () => {
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass', ask: ['write_*'] },
|
||||
agents: [{ lazySubagentConfigs: [{}] }],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass' },
|
||||
agents: [{ lazySubagentConfigs: [{}] }],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test('does not match an approval rule outside the reachable tool surface', () => {
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass', ask: ['delete_*'] },
|
||||
agents: [{ tools: ['read_file'] }],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test('includes host-generated runtime tools in approval admission', () => {
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass', ask: ['check_background_task'] },
|
||||
agents: [{}],
|
||||
hostGeneratedToolNames: ['check_background_task'],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test('matches static and request-scoped rules against MCP aliases', () => {
|
||||
const agent: ToolApprovalAdmissionAgent = {
|
||||
tools: ['mcp__server__read_file'],
|
||||
mcpToolAliases: [{ name: 'mcp__server__read_file', aliasName: 'read_file' }],
|
||||
};
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass', ask: ['read_file'] },
|
||||
agents: [agent],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass' },
|
||||
agents: [agent],
|
||||
resolvedProgrammaticHooks: [{ hook: askHook, matcher: '^read_file$' }],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test('asks deployment hook sources only about concrete runtime tool names', () => {
|
||||
const hasToolApprovalHooks = jest.fn(
|
||||
(names?: readonly string[]) => names?.includes('write_file') === true,
|
||||
);
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, mode: 'bypass' },
|
||||
agents: [{ tools: ['read_file', 'write_file'] }],
|
||||
pluginHookSource: pluginSource(hasToolApprovalHooks),
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(hasToolApprovalHooks).toHaveBeenCalledWith(['read_file']);
|
||||
expect(hasToolApprovalHooks).toHaveBeenCalledWith(['write_file']);
|
||||
});
|
||||
|
||||
test('classifies top-level ask_user_question unless it is filtered or denied', () => {
|
||||
const agents = [{ tools: ['ask_user_question'] }];
|
||||
expect(canAgentGraphPause({ policy: undefined, agents })).toBe(true);
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true, deny: ['ask_*'] },
|
||||
agents,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: { enabled: true },
|
||||
agents,
|
||||
askUserQuestionAdminDisabled: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test('does not promote nested ask_user_question to a parent pause capability', () => {
|
||||
expect(
|
||||
canAgentGraphPause({
|
||||
policy: undefined,
|
||||
agents: [{ subagentAgentConfigs: [{ tools: ['ask_user_question'] }] }],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('agentRunUsesCheckpointer', () => {
|
||||
test('tracks checkpointer attachment independently from current pause capability', () => {
|
||||
const policy = { enabled: true, mode: 'bypass' as const };
|
||||
const agents = [{ tools: ['read_file'] }];
|
||||
expect(canAgentGraphPause({ policy, agents })).toBe(false);
|
||||
expect(agentRunUsesCheckpointer({ policy, agents })).toBe(true);
|
||||
});
|
||||
|
||||
test('uses the same top-level ask-tool admin gate as createRun', () => {
|
||||
const agents = [{ tools: ['ask_user_question'] }];
|
||||
expect(agentRunUsesCheckpointer({ policy: undefined, agents })).toBe(true);
|
||||
expect(
|
||||
agentRunUsesCheckpointer({
|
||||
policy: undefined,
|
||||
agents,
|
||||
askUserQuestionAdminDisabled: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
185
packages/api/src/agents/hitl/admission.ts
Normal file
185
packages/api/src/agents/hitl/admission.ts
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
import type { TToolApprovalPolicy } from 'librechat-data-provider';
|
||||
import type { PluginHookSource } from '~/agents/hooks/source';
|
||||
import type { MCPToolAlias } from '~/tools/classification';
|
||||
import type { ResolvedToolApprovalHook } from './hooks';
|
||||
import {
|
||||
healToolApprovalPolicy,
|
||||
isHITLEnabled,
|
||||
isToolApprovalPauseCapable,
|
||||
isToolDeniedByApprovalPolicy,
|
||||
} from './policy';
|
||||
import { ASK_USER_QUESTION_TOOL_NAME } from './askUserQuestionTool';
|
||||
import { resolvedToolApprovalHooksCanMatch } from './hooks';
|
||||
|
||||
interface ApprovalToolReference {
|
||||
readonly name?: string;
|
||||
}
|
||||
|
||||
interface ApprovalToolRegistry {
|
||||
keys(): Iterable<string>;
|
||||
has(name: string): boolean;
|
||||
}
|
||||
|
||||
interface ApprovalSubagentGraph {
|
||||
readonly memberConfigs?: readonly (ToolApprovalAdmissionAgent | null | undefined)[];
|
||||
}
|
||||
|
||||
export interface ToolApprovalAdmissionAgent {
|
||||
readonly tools?: readonly (string | ApprovalToolReference)[];
|
||||
readonly toolRegistry?: ApprovalToolRegistry;
|
||||
readonly toolDefinitions?: readonly ApprovalToolReference[];
|
||||
readonly mcpToolAliases?: readonly MCPToolAlias[];
|
||||
readonly subagentAgentConfigs?: readonly (ToolApprovalAdmissionAgent | null | undefined)[];
|
||||
readonly lazySubagentConfigs?: readonly (ToolApprovalAdmissionAgent | null | undefined)[];
|
||||
readonly subagentGraphMemberMetadata?: readonly (ToolApprovalAdmissionAgent | null | undefined)[];
|
||||
readonly subagentGraphConfigs?: readonly ApprovalSubagentGraph[];
|
||||
}
|
||||
|
||||
export interface ToolApprovalAdmissionInput {
|
||||
readonly policy: TToolApprovalPolicy | undefined;
|
||||
readonly agents: readonly (ToolApprovalAdmissionAgent | null | undefined)[];
|
||||
readonly hostGeneratedToolNames?: readonly string[];
|
||||
readonly resolvedProgrammaticHooks?: readonly ResolvedToolApprovalHook[];
|
||||
readonly pluginHookSource?: PluginHookSource;
|
||||
readonly askUserQuestionAdminDisabled?: boolean;
|
||||
}
|
||||
|
||||
function agentHasTool(agent: ToolApprovalAdmissionAgent, toolName: string): boolean {
|
||||
return (
|
||||
agent.tools?.some((tool) => (typeof tool === 'string' ? tool : tool.name) === toolName) ===
|
||||
true ||
|
||||
agent.toolRegistry?.has(toolName) === true ||
|
||||
agent.toolDefinitions?.some((definition) => definition.name === toolName) === true
|
||||
);
|
||||
}
|
||||
|
||||
function collectApprovalAgents(roots: readonly (ToolApprovalAdmissionAgent | null | undefined)[]): {
|
||||
agents: ToolApprovalAdmissionAgent[];
|
||||
hasLazyToolSurface: boolean;
|
||||
} {
|
||||
const agents: ToolApprovalAdmissionAgent[] = [];
|
||||
const visited = new Set<ToolApprovalAdmissionAgent>();
|
||||
const pending = [...roots];
|
||||
let hasLazyToolSurface = false;
|
||||
|
||||
for (let index = 0; index < pending.length; index++) {
|
||||
const agent = pending[index];
|
||||
if (agent == null || visited.has(agent)) {
|
||||
continue;
|
||||
}
|
||||
visited.add(agent);
|
||||
agents.push(agent);
|
||||
pending.push(...(agent.subagentAgentConfigs ?? []));
|
||||
if ((agent.lazySubagentConfigs?.length ?? 0) > 0) {
|
||||
hasLazyToolSurface = true;
|
||||
pending.push(...(agent.lazySubagentConfigs ?? []));
|
||||
}
|
||||
pending.push(...(agent.subagentGraphMemberMetadata ?? []));
|
||||
for (const graph of agent.subagentGraphConfigs ?? []) {
|
||||
pending.push(...(graph.memberConfigs ?? []));
|
||||
}
|
||||
}
|
||||
|
||||
return { agents, hasLazyToolSurface };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an initialized run can pause through tool approval or a top-level
|
||||
* `ask_user_question`. Eager tools are matched exactly across every subagent
|
||||
* form; unresolved lazy surfaces are classified conservatively. The interrupt
|
||||
* boundary remains the final fail-closed durability check.
|
||||
*/
|
||||
export function canAgentGraphPause({
|
||||
policy,
|
||||
agents,
|
||||
hostGeneratedToolNames = [],
|
||||
resolvedProgrammaticHooks = [],
|
||||
pluginHookSource,
|
||||
askUserQuestionAdminDisabled = false,
|
||||
}: ToolApprovalAdmissionInput): boolean {
|
||||
const asksUserQuestion =
|
||||
!askUserQuestionAdminDisabled &&
|
||||
!isToolDeniedByApprovalPolicy(policy, ASK_USER_QUESTION_TOOL_NAME) &&
|
||||
agents.some((agent) => agent != null && agentHasTool(agent, ASK_USER_QUESTION_TOOL_NAME));
|
||||
if (!isHITLEnabled(policy)) {
|
||||
return asksUserQuestion;
|
||||
}
|
||||
|
||||
const approvalGraph = collectApprovalAgents(agents);
|
||||
const toolNames = new Set<string>();
|
||||
const aliases: MCPToolAlias[] = [];
|
||||
const aliasesByToolName = new Map<string, string[]>();
|
||||
const addToolName = (name: unknown) => {
|
||||
if (typeof name === 'string' && name !== ASK_USER_QUESTION_TOOL_NAME) {
|
||||
toolNames.add(name);
|
||||
}
|
||||
};
|
||||
|
||||
for (const name of hostGeneratedToolNames) {
|
||||
addToolName(name);
|
||||
}
|
||||
|
||||
for (const agent of approvalGraph.agents) {
|
||||
for (const tool of agent.tools ?? []) {
|
||||
addToolName(typeof tool === 'string' ? tool : tool.name);
|
||||
}
|
||||
if (agent.toolRegistry) {
|
||||
for (const name of agent.toolRegistry.keys()) {
|
||||
addToolName(name);
|
||||
}
|
||||
}
|
||||
for (const definition of agent.toolDefinitions ?? []) {
|
||||
addToolName(definition.name);
|
||||
}
|
||||
for (const alias of agent.mcpToolAliases ?? []) {
|
||||
aliases.push(alias);
|
||||
const names = aliasesByToolName.get(alias.name) ?? [];
|
||||
names.push(alias.aliasName);
|
||||
aliasesByToolName.set(alias.name, names);
|
||||
}
|
||||
}
|
||||
|
||||
const effectivePolicy = healToolApprovalPolicy(policy, aliases);
|
||||
const knownToolCanPause = Array.from(toolNames).some((toolName) => {
|
||||
const matcherNames = [toolName, ...(aliasesByToolName.get(toolName) ?? [])];
|
||||
const requestHookCanAsk = resolvedToolApprovalHooksCanMatch(
|
||||
resolvedProgrammaticHooks,
|
||||
matcherNames,
|
||||
);
|
||||
const pluginHookCanAsk = pluginHookSource?.hasToolApprovalHooks?.([toolName]) === true;
|
||||
return isToolApprovalPauseCapable(effectivePolicy, requestHookCanAsk || pluginHookCanAsk, [
|
||||
toolName,
|
||||
]);
|
||||
});
|
||||
if (knownToolCanPause) {
|
||||
return true;
|
||||
}
|
||||
if (approvalGraph.hasLazyToolSurface) {
|
||||
const unresolvedHookCanAsk =
|
||||
resolvedProgrammaticHooks.length > 0 || pluginHookSource?.hasToolApprovalHooks?.() === true;
|
||||
if (isToolApprovalPauseCapable(effectivePolicy, unresolvedHookCanAsk)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return asksUserQuestion;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `createRun` attaches a checkpointer for this initialization.
|
||||
* Cleanup follows attachment, not current pause capability: a retry must not
|
||||
* restore remnants written before a policy or request-hook change.
|
||||
*/
|
||||
export function agentRunUsesCheckpointer({
|
||||
policy,
|
||||
agents,
|
||||
askUserQuestionAdminDisabled = false,
|
||||
}: Pick<
|
||||
ToolApprovalAdmissionInput,
|
||||
'policy' | 'agents' | 'askUserQuestionAdminDisabled'
|
||||
>): boolean {
|
||||
return (
|
||||
isHITLEnabled(policy) ||
|
||||
(!askUserQuestionAdminDisabled &&
|
||||
agents.some((agent) => agent != null && agentHasTool(agent, ASK_USER_QUESTION_TOOL_NAME)))
|
||||
);
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import {
|
|||
getRegisteredToolApprovalHookCount,
|
||||
clearToolApprovalHooks,
|
||||
buildToolApprovalHooks,
|
||||
resolvedToolApprovalHooksCanMatch,
|
||||
} from './hooks';
|
||||
|
||||
const denyHook: ToolApprovalHook = async () => ({ decision: 'deny' });
|
||||
|
|
@ -74,4 +75,18 @@ describe('tool-approval hook registry', () => {
|
|||
expect(buildToolApprovalHooks({})).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolvedToolApprovalHooksCanMatch', () => {
|
||||
test('matches only hooks that can run for the selected tool names', () => {
|
||||
const hooks = [{ hook: denyHook, matcher: '^approval_probe$' }];
|
||||
expect(resolvedToolApprovalHooksCanMatch(hooks, ['read_file'])).toBe(false);
|
||||
expect(resolvedToolApprovalHooksCanMatch(hooks, ['approval_probe'])).toBe(true);
|
||||
});
|
||||
|
||||
test('treats an unscoped hook as matching any nonempty tool surface', () => {
|
||||
const hooks = [{ hook: denyHook }];
|
||||
expect(resolvedToolApprovalHooksCanMatch(hooks, [])).toBe(false);
|
||||
expect(resolvedToolApprovalHooksCanMatch(hooks, ['read_file'])).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -44,6 +44,11 @@ export type ToolApprovalHookFactory = (
|
|||
context: ToolApprovalHookContext,
|
||||
) => ToolApprovalHook | undefined;
|
||||
|
||||
export interface ResolvedToolApprovalHook {
|
||||
hook: ToolApprovalHook;
|
||||
matcher?: string;
|
||||
}
|
||||
|
||||
interface RegisteredHook {
|
||||
factory: ToolApprovalHookFactory;
|
||||
/** Optional regex matched against the tool name (the `PreToolUse` matcher `pattern`). */
|
||||
|
|
@ -106,8 +111,8 @@ export function clearToolApprovalHooks(): void {
|
|||
*/
|
||||
export function buildToolApprovalHooks(
|
||||
context: ToolApprovalHookContext,
|
||||
): Array<{ hook: ToolApprovalHook; matcher?: string }> {
|
||||
const built: Array<{ hook: ToolApprovalHook; matcher?: string }> = [];
|
||||
): ResolvedToolApprovalHook[] {
|
||||
const built: ResolvedToolApprovalHook[] = [];
|
||||
for (const { factory, matcher } of registeredHooks) {
|
||||
const hook = factory(context);
|
||||
if (hook) {
|
||||
|
|
@ -116,3 +121,25 @@ export function buildToolApprovalHooks(
|
|||
}
|
||||
return built;
|
||||
}
|
||||
|
||||
/** Whether any resolved hook matcher can run for one of the supplied tool names. */
|
||||
export function resolvedToolApprovalHooksCanMatch(
|
||||
hooks: readonly ResolvedToolApprovalHook[],
|
||||
toolNames: readonly string[],
|
||||
): boolean {
|
||||
return hooks.some(({ matcher }) => {
|
||||
if (matcher == null) {
|
||||
return toolNames.length > 0;
|
||||
}
|
||||
let regex: RegExp;
|
||||
try {
|
||||
regex = new RegExp(matcher);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return toolNames.some((name) => {
|
||||
regex.lastIndex = 0;
|
||||
return regex.test(name);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
export * from './policy';
|
||||
export * from './admission';
|
||||
export * from './runtime';
|
||||
export * from './resume';
|
||||
export * from './preflight';
|
||||
|
|
|
|||
|
|
@ -17,8 +17,60 @@ import {
|
|||
applyResumeContext,
|
||||
applyResumeModelParameters,
|
||||
exemptAskUserQuestionFromApproval,
|
||||
isToolApprovalPauseCapable,
|
||||
isToolDeniedByApprovalPolicy,
|
||||
} from './policy';
|
||||
|
||||
describe('isToolApprovalPauseCapable', () => {
|
||||
it('recognizes the default ask fallback and explicit ask rules', () => {
|
||||
expect(isToolApprovalPauseCapable({ enabled: true })).toBe(true);
|
||||
expect(isToolApprovalPauseCapable({ enabled: true, mode: 'bypass', ask: ['write_*'] })).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('excludes policies that can only allow or deny', () => {
|
||||
expect(isToolApprovalPauseCapable({ enabled: true, mode: 'bypass' })).toBe(false);
|
||||
expect(isToolApprovalPauseCapable({ enabled: true, mode: 'dontAsk' })).toBe(false);
|
||||
expect(isToolApprovalPauseCapable({ enabled: true, allow: ['*'] })).toBe(false);
|
||||
expect(isToolApprovalPauseCapable({ enabled: true, deny: ['*'], ask: ['write_*'] }, true)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('treats a programmatic hook as pause-capable when policy does not deny every tool', () => {
|
||||
expect(isToolApprovalPauseCapable({ enabled: true, mode: 'bypass' }, true)).toBe(true);
|
||||
});
|
||||
|
||||
it('intersects approval rules with the selected run tool surface', () => {
|
||||
const policy = { enabled: true, mode: 'bypass' as const, ask: ['write_*'] };
|
||||
expect(isToolApprovalPauseCapable(policy, false, [])).toBe(false);
|
||||
expect(isToolApprovalPauseCapable(policy, false, ['read_file'])).toBe(false);
|
||||
expect(isToolApprovalPauseCapable(policy, false, ['write_file'])).toBe(true);
|
||||
expect(isToolApprovalPauseCapable({ enabled: true }, false, ['read_file'])).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps deny precedence when a matching programmatic hook can ask', () => {
|
||||
const policy = { enabled: true, mode: 'bypass' as const, deny: ['delete_*'] };
|
||||
expect(isToolApprovalPauseCapable(policy, true, ['delete_file'])).toBe(false);
|
||||
expect(isToolApprovalPauseCapable(policy, true, ['write_file'])).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isToolDeniedByApprovalPolicy', () => {
|
||||
it('matches exact and wildcard denies only when approval is enabled', () => {
|
||||
expect(
|
||||
isToolDeniedByApprovalPolicy({ enabled: true, deny: ['ask_*'] }, 'ask_user_question'),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isToolDeniedByApprovalPolicy({ enabled: true, deny: ['write_*'] }, 'ask_user_question'),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isToolDeniedByApprovalPolicy({ enabled: false, deny: ['ask_*'] }, 'ask_user_question'),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveToolApprovalPolicy', () => {
|
||||
test('returns the endpoint policy unchanged (single layer wired today)', () => {
|
||||
const endpoint: TToolApprovalPolicy = { enabled: true, mode: 'default', deny: ['rm'] };
|
||||
|
|
@ -263,6 +315,17 @@ describe('buildPendingAction', () => {
|
|||
expect(action.expiresAt).toBeGreaterThanOrEqual(before);
|
||||
expect(action.expiresAt).toBeLessThanOrEqual(after);
|
||||
});
|
||||
|
||||
test('caps the TTL at an inherited absolute deadline without a second clock read', () => {
|
||||
const deadline = Date.now() + 1_000;
|
||||
const action = buildPendingAction(toolApprovalPayload, {
|
||||
...ctx,
|
||||
ttlMs: 5_000,
|
||||
expiresAt: new Date(deadline),
|
||||
});
|
||||
|
||||
expect(action.expiresAt).toBe(deadline);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toClientPendingAction', () => {
|
||||
|
|
|
|||
|
|
@ -76,10 +76,63 @@ export function resolveToolApprovalPolicy(
|
|||
* any multi-process deployment. Pair this predicate with the checkpointer
|
||||
* assignment at the `Run.create` call site.
|
||||
*/
|
||||
export function isHITLEnabled(policy: TToolApprovalPolicy | undefined): boolean {
|
||||
export function isHITLEnabled(
|
||||
policy: TToolApprovalPolicy | undefined,
|
||||
): policy is NonNullable<TToolApprovalPolicy> {
|
||||
return policy?.enabled === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the configured policy can structurally return `ask` for any tool.
|
||||
*
|
||||
* `bypass` and `dontAsk` are non-pausing fallbacks; only an explicit ask rule
|
||||
* or a programmatic hook can tighten them to `ask`. A catch-all deny remains
|
||||
* non-pausing because deny wins over every hook/rule, while a catch-all allow
|
||||
* removes the default mode's unmatched-tool ask fallback. More-specific pattern
|
||||
* overlap is intentionally treated conservatively because the run's complete
|
||||
* lazy tool surface is not known at admission time.
|
||||
*/
|
||||
export function isToolApprovalPauseCapable(
|
||||
policy: TToolApprovalPolicy | undefined,
|
||||
hasProgrammaticHooks = false,
|
||||
toolNames?: readonly string[],
|
||||
): boolean {
|
||||
if (!isHITLEnabled(policy)) {
|
||||
return false;
|
||||
}
|
||||
const enabledPolicy = policy;
|
||||
if (toolNames != null) {
|
||||
const names = Array.from(new Set(toolNames.filter((name) => name.length > 0)));
|
||||
if (names.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const matches = (patterns: string[] | undefined, name: string): boolean =>
|
||||
patterns?.some((pattern) => globToRegex(pattern).test(name)) === true;
|
||||
return names.some((name) => {
|
||||
if (matches(enabledPolicy.deny, name)) {
|
||||
return false;
|
||||
}
|
||||
if (hasProgrammaticHooks || matches(enabledPolicy.ask, name)) {
|
||||
return true;
|
||||
}
|
||||
if (matches(enabledPolicy.allow, name)) {
|
||||
return false;
|
||||
}
|
||||
return enabledPolicy.mode !== 'bypass' && enabledPolicy.mode !== 'dontAsk';
|
||||
});
|
||||
}
|
||||
if (policy?.deny?.includes('*')) {
|
||||
return false;
|
||||
}
|
||||
if (hasProgrammaticHooks || (policy?.ask?.length ?? 0) > 0) {
|
||||
return true;
|
||||
}
|
||||
if (policy?.mode === 'bypass' || policy?.mode === 'dontAsk') {
|
||||
return false;
|
||||
}
|
||||
return policy?.allow?.includes('*') !== true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a LibreChat tool-approval policy to the SDK's `ToolPolicyConfig`.
|
||||
*
|
||||
|
|
@ -93,6 +146,17 @@ function globToRegex(pattern: string): RegExp {
|
|||
return new RegExp('^' + escaped.replace(/\*/g, '.*') + '$');
|
||||
}
|
||||
|
||||
/** Whether an enabled static policy unconditionally denies one concrete tool name. */
|
||||
export function isToolDeniedByApprovalPolicy(
|
||||
policy: TToolApprovalPolicy | undefined,
|
||||
toolName: string,
|
||||
): boolean {
|
||||
return (
|
||||
isHITLEnabled(policy) &&
|
||||
policy.deny?.some((pattern) => globToRegex(pattern).test(toolName)) === true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extends each `toolApproval` pattern list with the names of tools whose
|
||||
* OTHER spelling matches, so admin YAML keeps applying when a tool's key
|
||||
|
|
@ -252,6 +316,8 @@ export interface PendingActionContext {
|
|||
responseMessageId?: string;
|
||||
/** Optional TTL (ms). When set, `expiresAt = createdAt + ttlMs`. */
|
||||
ttlMs?: number;
|
||||
/** Optional absolute upper bound inherited from an enclosing event binding. */
|
||||
expiresAt?: Date | string | number;
|
||||
/** Override actionId; defaults to a fresh uuid. */
|
||||
actionId?: string;
|
||||
/** SDK interrupt id (`RunInterruptResult.interruptId`) for cross-process resume. */
|
||||
|
|
@ -661,6 +727,25 @@ export function buildPendingAction(
|
|||
ctx: PendingActionContext,
|
||||
): Agents.PendingAction {
|
||||
const createdAt = Date.now();
|
||||
const ttlExpiresAt = typeof ctx.ttlMs === 'number' ? createdAt + ctx.ttlMs : undefined;
|
||||
let absoluteExpiresAt: number | undefined;
|
||||
if (typeof ctx.expiresAt === 'number') {
|
||||
absoluteExpiresAt = ctx.expiresAt;
|
||||
} else if (ctx.expiresAt instanceof Date) {
|
||||
absoluteExpiresAt = ctx.expiresAt.getTime();
|
||||
} else if (typeof ctx.expiresAt === 'string') {
|
||||
absoluteExpiresAt = new Date(ctx.expiresAt).getTime();
|
||||
}
|
||||
const finiteAbsoluteExpiresAt = Number.isFinite(absoluteExpiresAt)
|
||||
? absoluteExpiresAt
|
||||
: undefined;
|
||||
let expiresAt = ttlExpiresAt;
|
||||
if (finiteAbsoluteExpiresAt != null) {
|
||||
expiresAt =
|
||||
ttlExpiresAt == null
|
||||
? finiteAbsoluteExpiresAt
|
||||
: Math.min(ttlExpiresAt, finiteAbsoluteExpiresAt);
|
||||
}
|
||||
return {
|
||||
actionId: ctx.actionId ?? randomUUID(),
|
||||
streamId: ctx.streamId,
|
||||
|
|
@ -669,7 +754,7 @@ export function buildPendingAction(
|
|||
responseMessageId: ctx.responseMessageId,
|
||||
payload,
|
||||
createdAt,
|
||||
expiresAt: typeof ctx.ttlMs === 'number' ? createdAt + ctx.ttlMs : undefined,
|
||||
expiresAt,
|
||||
interruptId: ctx.interruptId,
|
||||
threadId: ctx.threadId,
|
||||
requestFingerprint: ctx.requestFingerprint,
|
||||
|
|
|
|||
|
|
@ -88,6 +88,19 @@ describe('buildHITLRunWiring host-hook composition', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test('reuses request-scoped hooks resolved by admission without invoking factories twice', () => {
|
||||
const hook = async () => ({ decision: 'ask' as const });
|
||||
const factory = jest.fn(() => hook);
|
||||
registerToolApprovalHook(factory);
|
||||
const resolved = [{ hook }];
|
||||
factory.mockClear();
|
||||
|
||||
const wiring = buildHITLRunWiring({ enabled: true }, {}, [], resolved);
|
||||
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
expect(wiring?.hooks.getMatchers('PreToolUse')).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('matches lazy aliases without changing host-hook ordering', async () => {
|
||||
const hook = jest.fn(async () => ({ decision: 'deny' as const }));
|
||||
registerToolApprovalHook(() => hook, {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { HookRegistry, createToolPolicyHook } from '@librechat/agents';
|
||||
import type { TToolApprovalPolicy } from 'librechat-data-provider';
|
||||
import type { ResolvedToolApprovalHook, ToolApprovalHookContext } from './hooks';
|
||||
import type { MCPToolAlias } from '~/tools/classification';
|
||||
import type { ToolApprovalHookContext } from './hooks';
|
||||
import { isHITLEnabled, mapToolApprovalPolicy } from './policy';
|
||||
import { buildToolApprovalHooks } from './hooks';
|
||||
|
||||
|
|
@ -40,13 +40,14 @@ export function buildHITLRunWiring(
|
|||
policy: TToolApprovalPolicy | undefined,
|
||||
context: ToolApprovalHookContext = {},
|
||||
mcpToolAliases: readonly MCPToolAlias[] = [],
|
||||
resolvedProgrammaticHooks?: readonly ResolvedToolApprovalHook[],
|
||||
): HITLRunWiring | undefined {
|
||||
if (!isHITLEnabled(policy)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const registry = new HookRegistry();
|
||||
let activePolicy = policy;
|
||||
let activePolicy: TToolApprovalPolicy | undefined = policy;
|
||||
const aliases = [...mcpToolAliases];
|
||||
const registeredAliases = new Set(
|
||||
aliases.map(({ name, aliasName }) => `${name}\u0000${aliasName}`),
|
||||
|
|
@ -60,7 +61,7 @@ export function buildHITLRunWiring(
|
|||
});
|
||||
|
||||
// Host-registered programmatic hooks — context-aware, layered after the static-policy hook.
|
||||
const programmaticHooks = buildToolApprovalHooks(context);
|
||||
const programmaticHooks = resolvedProgrammaticHooks ?? buildToolApprovalHooks(context);
|
||||
for (const { hook, matcher } of programmaticHooks) {
|
||||
if (matcher == null) {
|
||||
registry.register('PreToolUse', { hooks: [hook] });
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ export interface PluginHookSourceOptions {
|
|||
*/
|
||||
export interface PluginHookSource {
|
||||
hasHooks(): boolean;
|
||||
/** Whether a ready deployment hook can participate in tool-approval decisions. */
|
||||
hasToolApprovalHooks?(toolNames?: readonly string[]): boolean;
|
||||
register(options: PluginHookSourceOptions): number;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import type { Callbacks } from '@langchain/core/callbacks/manager';
|
|||
import type { AppConfig, IUser } from '@librechat/data-schemas';
|
||||
import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent';
|
||||
import type { ToolInputValidationError } from '~/agents/toolValidation';
|
||||
import type { ResolvedToolApprovalHook } from '~/agents/hitl/hooks';
|
||||
import type { ResolvedAlwaysApplySkill } from '~/agents/skills';
|
||||
import type { MCPToolAlias } from '~/tools/classification';
|
||||
import type { SubagentUsageEvent } from '~/agents/usage';
|
||||
|
|
@ -1091,7 +1092,7 @@ export function agentRequestsAskUserQuestion(agent: {
|
|||
* it to the model, attaching checkpointers, and pausing runs — for a run-pausing
|
||||
* tool the filter must be an actual kill switch.
|
||||
*/
|
||||
function isAskUserQuestionAdminDisabled(appConfig?: AppConfig): boolean {
|
||||
export function isAskUserQuestionAdminDisabled(appConfig?: AppConfig): boolean {
|
||||
const included = appConfig?.includedTools;
|
||||
if (included != null && included.length > 0) {
|
||||
return !included.includes(ASK_USER_QUESTION_TOOL_NAME);
|
||||
|
|
@ -1392,6 +1393,7 @@ export async function createRun({
|
|||
activityPhase,
|
||||
eventActorCheckpointing = false,
|
||||
hitlCapable = false,
|
||||
resolvedToolApprovalHooks,
|
||||
toolInputValidationErrors,
|
||||
sessionStartSource,
|
||||
streaming = true,
|
||||
|
|
@ -1490,6 +1492,11 @@ export async function createRun({
|
|||
* final response / `[DONE]` with the tool call left unresolved).
|
||||
*/
|
||||
hitlCapable?: boolean;
|
||||
/**
|
||||
* Request-scoped approval hooks already resolved by the scheduled-run admission guard.
|
||||
* Reuse them here so a context-aware factory is evaluated exactly once for the run.
|
||||
*/
|
||||
resolvedToolApprovalHooks?: readonly ResolvedToolApprovalHook[];
|
||||
/** Plugin-hook SessionStart lifecycle source: 'startup' (default) or 'resume' on HITL-rebuild paths. */
|
||||
sessionStartSource?: string;
|
||||
/** Request-scoped tool input failures consumed by the completion handler. */
|
||||
|
|
@ -1881,6 +1888,7 @@ export async function createRun({
|
|||
appConfig,
|
||||
},
|
||||
mcpToolAliases,
|
||||
resolvedToolApprovalHooks,
|
||||
)
|
||||
: undefined;
|
||||
registerResolvedMCPToolAliases = (resolvedAgent) => {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
getDeploymentPluginHookCapabilities,
|
||||
registerDeploymentPluginHooks,
|
||||
hasDeploymentPluginHooks,
|
||||
hasDeploymentPluginToolApprovalHooks,
|
||||
} from './runtime';
|
||||
import { getPluginHookSource, setPluginHookSource } from '~/agents/hooks';
|
||||
import { initializeDeploymentPlugins } from './deployment';
|
||||
|
|
@ -306,6 +307,9 @@ describe('registerDeploymentPluginHooks', () => {
|
|||
});
|
||||
const source = getPluginHookSource();
|
||||
expect(source?.hasHooks()).toBe(true);
|
||||
expect(hasDeploymentPluginToolApprovalHooks()).toBe(true);
|
||||
expect(hasDeploymentPluginToolApprovalHooks(['unmatched_tool'])).toBe(false);
|
||||
expect(hasDeploymentPluginToolApprovalHooks(['write_file'])).toBe(true);
|
||||
const registry = new HookRegistry();
|
||||
expect(source?.register({ registry, context: { sessionId: 'conversation-seam' } })).toBe(4);
|
||||
setPluginHookSource(undefined);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { matchesQuery } from '@librechat/agents';
|
||||
import type { HookRegistry, HookOutput } from '@librechat/agents';
|
||||
import type {
|
||||
PluginHookExecutionRequest,
|
||||
|
|
@ -113,6 +114,18 @@ export function hasDeploymentPluginHooks(): boolean {
|
|||
return getExecutableHookPlugins().length > 0;
|
||||
}
|
||||
|
||||
/** Whether any executable deployment-plugin hook can return a tool decision. */
|
||||
export function hasDeploymentPluginToolApprovalHooks(toolNames?: readonly string[]): boolean {
|
||||
return getExecutableHookPlugins().some((plugin) =>
|
||||
plugin.hooks?.plan.entries.some((entry) => {
|
||||
if (entry.status !== 'ready' || entry.targetEvent !== 'PreToolUse') {
|
||||
return false;
|
||||
}
|
||||
return toolNames == null || toolNames.some((name) => matchesQuery(entry.matcher, name));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export interface RegisterDeploymentPluginHooksOptions {
|
||||
registry: HookRegistry;
|
||||
context?: PluginHookRuntimeContext;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import type {
|
|||
} from '~/stream/interfaces/IJobStore';
|
||||
import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';
|
||||
import {
|
||||
JobStatusTransitionDeadlineError,
|
||||
isPendingActionExpired,
|
||||
isPendingActionStale,
|
||||
PAUSE_PERSISTENCE_TIMEOUT_ERROR,
|
||||
|
|
@ -35,6 +36,17 @@ export interface ApprovalPauseOptions {
|
|||
persistencePending?: boolean;
|
||||
}
|
||||
|
||||
export const PENDING_ACTION_EXPIRED_CODE = 'HITL_ACTION_EXPIRED';
|
||||
|
||||
export class PendingActionExpiredError extends Error {
|
||||
readonly code: typeof PENDING_ACTION_EXPIRED_CODE = PENDING_ACTION_EXPIRED_CODE;
|
||||
|
||||
constructor() {
|
||||
super('The pending action expired before it could be exposed for review');
|
||||
this.name = 'PendingActionExpiredError';
|
||||
}
|
||||
}
|
||||
|
||||
const PAUSE_PERSISTENCE_ACTION_PREFIX = 'pause-persistence:';
|
||||
|
||||
export function pausePersistenceActionId(actionId: string): string {
|
||||
|
|
@ -85,6 +97,9 @@ export class ApprovalLifecycle {
|
|||
pendingAction: Agents.PendingAction,
|
||||
options: ApprovalPauseOptions = {},
|
||||
): Promise<boolean> {
|
||||
if (isPendingActionExpired({ pendingAction })) {
|
||||
throw new PendingActionExpiredError();
|
||||
}
|
||||
const job = await this.store.getJob(streamId);
|
||||
if (
|
||||
!job ||
|
||||
|
|
@ -107,28 +122,37 @@ export class ApprovalLifecycle {
|
|||
: 0,
|
||||
);
|
||||
const persistenceStartedAt = Date.now();
|
||||
const ok = await this.store.transitionStatus(streamId, {
|
||||
from: 'running',
|
||||
to: 'requires_action',
|
||||
// pendingActionId is the flat mirror the atomic resolve/expire guard on.
|
||||
patch: {
|
||||
pendingAction,
|
||||
pendingActionId:
|
||||
options.persistencePending === true
|
||||
? pausePersistenceActionId(pendingAction.actionId)
|
||||
: pendingAction.actionId,
|
||||
...(options.persistencePending === true && {
|
||||
terminalPersistencePending: true,
|
||||
terminalPersistenceStartedAt: persistenceStartedAt,
|
||||
}),
|
||||
...(discoveredTools != null && discoveredTools.length > 0
|
||||
? { discoveredTools: [...discoveredTools] }
|
||||
: {}),
|
||||
...(activityPhaseSnapshot != null ? { activityPhaseSnapshot } : {}),
|
||||
},
|
||||
expectCreatedAt: expectedCreatedAt,
|
||||
steerReceiptTtlSeconds: pauseReceiptTtl,
|
||||
});
|
||||
let ok: boolean;
|
||||
try {
|
||||
ok = await this.store.transitionStatus(streamId, {
|
||||
from: 'running',
|
||||
to: 'requires_action',
|
||||
// pendingActionId is the flat mirror the atomic resolve/expire guard on.
|
||||
patch: {
|
||||
pendingAction,
|
||||
pendingActionId:
|
||||
options.persistencePending === true
|
||||
? pausePersistenceActionId(pendingAction.actionId)
|
||||
: pendingAction.actionId,
|
||||
...(options.persistencePending === true && {
|
||||
terminalPersistencePending: true,
|
||||
terminalPersistenceStartedAt: persistenceStartedAt,
|
||||
}),
|
||||
...(discoveredTools != null && discoveredTools.length > 0
|
||||
? { discoveredTools: [...discoveredTools] }
|
||||
: {}),
|
||||
...(activityPhaseSnapshot != null ? { activityPhaseSnapshot } : {}),
|
||||
},
|
||||
expectCreatedAt: expectedCreatedAt,
|
||||
notAfterMs: pendingAction.expiresAt,
|
||||
steerReceiptTtlSeconds: pauseReceiptTtl,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof JobStatusTransitionDeadlineError) {
|
||||
throw new PendingActionExpiredError();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (ok) {
|
||||
this.callbacks.onPaused?.(streamId, expectedCreatedAt);
|
||||
logger.debug(
|
||||
|
|
|
|||
|
|
@ -200,9 +200,9 @@ describe('RedisJobStore', () => {
|
|||
}),
|
||||
).resolves.toBe(false);
|
||||
|
||||
// ARGV[4] is the terminal job TTL. The persistence owner/recovery path
|
||||
// ARGV[5] is the terminal job TTL. The persistence owner/recovery path
|
||||
// gets five minutes even when ordinary completed records are immediate.
|
||||
expect(evalTransition.mock.calls[0][15]).toBe('300');
|
||||
expect(evalTransition.mock.calls[0][16]).toBe('300');
|
||||
});
|
||||
|
||||
test('retains the generation epoch beyond the paused job TTL', async () => {
|
||||
|
|
@ -223,9 +223,9 @@ describe('RedisJobStore', () => {
|
|||
expect(transitionCall[0]).toContain(
|
||||
'redis.call("SET", KEYS[8], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
|
||||
);
|
||||
expect(transitionCall[15]).toBe('4621');
|
||||
expect(transitionCall[16]).toBe('0');
|
||||
expect(transitionCall[20]).toBe('300');
|
||||
expect(transitionCall[16]).toBe('4621');
|
||||
expect(transitionCall[17]).toBe('0');
|
||||
expect(transitionCall[21]).toBe('300');
|
||||
});
|
||||
|
||||
test('seeds the guarded epoch when reaping a legacy job without a marker', async () => {
|
||||
|
|
@ -991,9 +991,9 @@ describe('RedisJobStore', () => {
|
|||
evalTransition.mock.invocationCallOrder[0],
|
||||
);
|
||||
const transitionCall = evalTransition.mock.calls[0];
|
||||
expect(transitionCall[15]).toBe('86400');
|
||||
expect(transitionCall[17]).toBe('86400');
|
||||
expect(transitionCall[16]).toBe('86400');
|
||||
expect(transitionCall[18]).toBe('86400');
|
||||
expect(transitionCall[19]).toBe('86400');
|
||||
});
|
||||
|
||||
test('acknowledges evidence TTLs without removing a successor retry member', async () => {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import type { Agents } from 'librechat-data-provider';
|
|||
import type { Redis, Cluster } from 'ioredis';
|
||||
import type { SteerQueueItem, SteerReceipt } from '../interfaces/IJobStore';
|
||||
import {
|
||||
JobStatusTransitionDeadlineError,
|
||||
PAUSE_PERSISTENCE_TIMEOUT_ERROR,
|
||||
STEER_ENQUEUE_RECEIPT_FULL,
|
||||
} from '../interfaces/IJobStore';
|
||||
|
|
@ -129,6 +130,33 @@ describe('RedisJobStore Integration Tests', () => {
|
|||
await store.destroy();
|
||||
});
|
||||
|
||||
test('atomically rejects a status transition after its deadline', async () => {
|
||||
if (!ioredisClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { RedisJobStore } = await import('../implementations/RedisJobStore');
|
||||
const store = new RedisJobStore(ioredisClient);
|
||||
await store.initialize();
|
||||
|
||||
const streamId = `test-transition-deadline-${Date.now()}`;
|
||||
await store.createJob(streamId, 'deadline-user', streamId);
|
||||
|
||||
await expect(
|
||||
store.transitionStatus(streamId, {
|
||||
from: 'running',
|
||||
to: 'requires_action',
|
||||
notAfterMs: Date.now() - 1,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
name: JobStatusTransitionDeadlineError.name,
|
||||
notAfterMs: expect.any(Number),
|
||||
});
|
||||
await expect(store.getJob(streamId)).resolves.toMatchObject({ status: 'running' });
|
||||
|
||||
await store.destroy();
|
||||
});
|
||||
|
||||
test('returns the exact predecessor captured by the atomic replacement script', async () => {
|
||||
if (!ioredisClient) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import type { Agents } from 'librechat-data-provider';
|
||||
import { ApprovalLifecycle, PendingActionExpiredError } from '~/stream/ApprovalLifecycle';
|
||||
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
|
||||
import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
|
||||
import { PAUSE_PERSISTENCE_TIMEOUT_ERROR } from '~/stream/interfaces/IJobStore';
|
||||
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
|
||||
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
|
||||
import { ApprovalLifecycle } from '~/stream/ApprovalLifecycle';
|
||||
|
||||
jest.spyOn(console, 'log').mockImplementation();
|
||||
|
||||
|
|
@ -60,6 +60,42 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
|
|||
}
|
||||
});
|
||||
|
||||
test('refuses an action whose deadline passed before the pause transition', async () => {
|
||||
const streamId = 'stream-expired-before-pause';
|
||||
await manager.createJob(streamId, 'user-1');
|
||||
|
||||
await expect(
|
||||
manager.approvals.pause(streamId, buildAction(streamId, { expiresAt: Date.now() - 1 })),
|
||||
).rejects.toBeInstanceOf(PendingActionExpiredError);
|
||||
await expect(manager.getJob(streamId)).resolves.toMatchObject({ status: 'running' });
|
||||
await expect(manager.approvals.peek(streamId)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
test('refuses an action that expires while the pause transition is waiting on storage', async () => {
|
||||
const streamId = 'stream-expired-during-pause';
|
||||
await manager.createJob(streamId, 'user-1');
|
||||
const expiresAt = Date.now() + 1000;
|
||||
const originalTransition = jobStore.transitionStatus.bind(jobStore);
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt - 1);
|
||||
const transition = jest
|
||||
.spyOn(jobStore, 'transitionStatus')
|
||||
.mockImplementationOnce(async (...args) => {
|
||||
now.mockReturnValue(expiresAt);
|
||||
return originalTransition(...args);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
manager.approvals.pause(streamId, buildAction(streamId, { expiresAt })),
|
||||
).rejects.toBeInstanceOf(PendingActionExpiredError);
|
||||
} finally {
|
||||
transition.mockRestore();
|
||||
now.mockRestore();
|
||||
}
|
||||
await expect(manager.getJob(streamId)).resolves.toMatchObject({ status: 'running' });
|
||||
await expect(manager.approvals.peek(streamId)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
test('a later ask retains a legacy answer for ordered cross-replica reconstruction', async () => {
|
||||
const streamId = 'stream-repause-retains-legacy-answer';
|
||||
await manager.createJob(streamId, 'user-1');
|
||||
|
|
@ -610,12 +646,15 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
|
|||
test('treats a past-expiresAt record as gone (lazy expiry)', async () => {
|
||||
const streamId = 'stream-expired-peek';
|
||||
await manager.createJob(streamId, 'user-1');
|
||||
await manager.approvals.pause(
|
||||
streamId,
|
||||
buildAction(streamId, { expiresAt: Date.now() - 1000 }),
|
||||
);
|
||||
const expiresAt = Date.now() + 1000;
|
||||
await manager.approvals.pause(streamId, buildAction(streamId, { expiresAt }));
|
||||
|
||||
expect(await manager.approvals.peek(streamId)).toBeNull();
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
|
||||
try {
|
||||
expect(await manager.approvals.peek(streamId)).toBeNull();
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -724,13 +763,16 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
|
|||
test('an expired pending action expires instead of resuming', async () => {
|
||||
const streamId = 'stream-resolve-expired';
|
||||
await manager.createJob(streamId, 'user-1');
|
||||
await manager.approvals.pause(
|
||||
streamId,
|
||||
buildAction(streamId, { expiresAt: Date.now() - 1000 }),
|
||||
);
|
||||
const expiresAt = Date.now() + 1000;
|
||||
await manager.approvals.pause(streamId, buildAction(streamId, { expiresAt }));
|
||||
|
||||
expect(await manager.approvals.resolve(streamId)).toBe(false);
|
||||
expect(await manager.getJobStatus(streamId)).toBe('aborted');
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
|
||||
try {
|
||||
expect(await manager.approvals.resolve(streamId)).toBe(false);
|
||||
expect(await manager.getJobStatus(streamId)).toBe('aborted');
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -813,13 +855,19 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
|
|||
scheduledFor: '2026-08-17T12:00:00.000Z',
|
||||
});
|
||||
const lifecycle = new ApprovalLifecycle(jobStore);
|
||||
await lifecycle.pause(streamId, buildAction(streamId, { expiresAt: Date.now() - 1 }));
|
||||
const expiresAt = Date.now() + 1_000;
|
||||
await lifecycle.pause(streamId, buildAction(streamId, { expiresAt }));
|
||||
const onApprovalExpired = jest.fn(async () => undefined);
|
||||
manager.setApprovalExpiredHandler(onApprovalExpired);
|
||||
|
||||
await (
|
||||
manager as unknown as { expireStaleApprovals: () => Promise<void> }
|
||||
).expireStaleApprovals();
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
|
||||
try {
|
||||
await (
|
||||
manager as unknown as { expireStaleApprovals: () => Promise<void> }
|
||||
).expireStaleApprovals();
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
|
||||
expect(onApprovalExpired).toHaveBeenCalledWith(
|
||||
streamId,
|
||||
|
|
@ -1182,13 +1230,16 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
|
|||
test('excludes a pending-approval job whose prompt has expired', async () => {
|
||||
const streamId = 'stream-expired-active';
|
||||
await manager.createJob(streamId, 'user-exp');
|
||||
await manager.approvals.pause(
|
||||
streamId,
|
||||
buildAction(streamId, { expiresAt: Date.now() - 1000 }),
|
||||
);
|
||||
const expiresAt = Date.now() + 1000;
|
||||
await manager.approvals.pause(streamId, buildAction(streamId, { expiresAt }));
|
||||
|
||||
// Still requires_action, but the prompt is past expiry → no longer active.
|
||||
expect(await manager.getActiveJobIdsForUser('user-exp')).not.toContain(streamId);
|
||||
const now = jest.spyOn(Date, 'now').mockReturnValue(expiresAt + 1);
|
||||
try {
|
||||
// Still requires_action, but the prompt is past expiry → no longer active.
|
||||
expect(await manager.getActiveJobIdsForUser('user-exp')).not.toContain(streamId);
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import type {
|
|||
} from '~/stream/interfaces/IJobStore';
|
||||
import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';
|
||||
import {
|
||||
JobStatusTransitionDeadlineError,
|
||||
JobPredecessorMismatchError,
|
||||
STEER_ENQUEUE_NOT_RUNNING,
|
||||
STEER_ENQUEUE_QUEUE_FULL,
|
||||
|
|
@ -770,6 +771,9 @@ export class InMemoryJobStore implements IJobStoreV2 {
|
|||
if (args.expectCreatedAt != null && job.createdAt !== args.expectCreatedAt) {
|
||||
return false;
|
||||
}
|
||||
if (args.notAfterMs != null && Date.now() >= args.notAfterMs) {
|
||||
throw new JobStatusTransitionDeadlineError(args.notAfterMs);
|
||||
}
|
||||
if (['complete', 'error', 'aborted'].includes(args.to)) {
|
||||
if (!this.isParkedRecoveryCompatible(streamId, job)) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';
|
|||
import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';
|
||||
import {
|
||||
JobCreationSupersededError,
|
||||
JobStatusTransitionDeadlineError,
|
||||
JobPredecessorMismatchError,
|
||||
STEER_ENQUEUE_NOT_RUNNING,
|
||||
STEER_QUEUE_MAX_DEPTH,
|
||||
|
|
@ -107,6 +108,7 @@ function assertCreateIdempotencyArguments(
|
|||
* from,
|
||||
* expectActionId | "",
|
||||
* expectCreatedAt | "",
|
||||
* notAfterMs | "",
|
||||
* ttl,
|
||||
* terminal ("0" | "1"),
|
||||
* chunksAfterComplete,
|
||||
|
|
@ -124,14 +126,17 @@ const JOB_CAS_LUA =
|
|||
'if redis.call("HGET", KEYS[1], "status") ~= ARGV[1] then return 0 end ' +
|
||||
'if ARGV[2] ~= "" and redis.call("HGET", KEYS[1], "pendingActionId") ~= ARGV[2] then return 0 end ' +
|
||||
'if ARGV[3] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[3] then return 0 end ' +
|
||||
'if ARGV[4] ~= "" then local now = redis.call("TIME") ' +
|
||||
'local nowMs = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) ' +
|
||||
'if nowMs >= tonumber(ARGV[4]) then return -1 end end ' +
|
||||
'local currentCreatedAt = redis.call("HGET", KEYS[1], "createdAt") ' +
|
||||
'local ttl = tonumber(ARGV[4]) ' +
|
||||
'local terminal = ARGV[5] == "1" ' +
|
||||
'local chunksTtl = tonumber(ARGV[6]) ' +
|
||||
'local runStepsTtl = tonumber(ARGV[7]) ' +
|
||||
'local parkedTtl = tonumber(ARGV[8]) ' +
|
||||
'local generationEpochGraceTtl = tonumber(ARGV[9]) ' +
|
||||
'local receiptTtl = tonumber(ARGV[10]) ' +
|
||||
'local ttl = tonumber(ARGV[5]) ' +
|
||||
'local terminal = ARGV[6] == "1" ' +
|
||||
'local chunksTtl = tonumber(ARGV[7]) ' +
|
||||
'local runStepsTtl = tonumber(ARGV[8]) ' +
|
||||
'local parkedTtl = tonumber(ARGV[9]) ' +
|
||||
'local generationEpochGraceTtl = tonumber(ARGV[10]) ' +
|
||||
'local receiptTtl = tonumber(ARGV[11]) ' +
|
||||
'local ownerUserId = redis.call("HGET", KEYS[1], "userId") ' +
|
||||
'local ownerTenantId = redis.call("HGET", KEYS[1], "tenantId") ' +
|
||||
'local generationProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" and 2 or 1 ' +
|
||||
|
|
@ -163,8 +168,8 @@ const JOB_CAS_LUA =
|
|||
'or (item.createdAt and (type(item.createdAt) ~= "number" or item.createdAt < 0)) ' +
|
||||
'or (item.recoveringCreatedAt and (type(item.recoveringCreatedAt) ~= "number" or item.recoveringCreatedAt < 0)) then return 0 end ' +
|
||||
'validatedPrior[#validatedPrior + 1] = item end end end ' +
|
||||
'local hdelCount = tonumber(ARGV[12]) ' +
|
||||
'local idx = 13 ' +
|
||||
'local hdelCount = tonumber(ARGV[13]) ' +
|
||||
'local idx = 14 ' +
|
||||
'for i = 1, hdelCount do redis.call("HDEL", KEYS[1], ARGV[idx]) idx = idx + 1 end ' +
|
||||
'local hset = {} ' +
|
||||
'for i = idx, #ARGV do hset[#hset + 1] = ARGV[i] end ' +
|
||||
|
|
@ -227,7 +232,7 @@ const JOB_CAS_LUA =
|
|||
'redis.call("DEL", KEYS[5], KEYS[6]) ' +
|
||||
'if chunksTtl == 0 then redis.call("DEL", KEYS[3]) else redis.call("EXPIRE", KEYS[3], chunksTtl) end ' +
|
||||
'if runStepsTtl == 0 then redis.call("DEL", KEYS[4]) else redis.call("EXPIRE", KEYS[4], runStepsTtl) end ' +
|
||||
'if ARGV[11] == "1" then if #items == 0 then return "[]" end return cjson.encode(items) end ' +
|
||||
'if ARGV[12] == "1" then if #items == 0 then return "[]" end return cjson.encode(items) end ' +
|
||||
'else ' +
|
||||
'redis.call("EXPIRE", KEYS[3], ttl) ' +
|
||||
'redis.call("EXPIRE", KEYS[4], ttl) ' +
|
||||
|
|
@ -2514,7 +2519,7 @@ export class RedisJobStore implements IJobStoreV2 {
|
|||
args: JobStatusTransition,
|
||||
returnDrainedSteers: boolean,
|
||||
): Promise<true | SteerQueueItem[] | null> {
|
||||
const { from, to, patch, clear, expectActionId, expectCreatedAt } = args;
|
||||
const { from, to, patch, clear, expectActionId, expectCreatedAt, notAfterMs } = args;
|
||||
const key = KEYS.job(streamId);
|
||||
|
||||
// status + patch become HSET pairs; serializeJob skips undefined, so
|
||||
|
|
@ -2573,6 +2578,7 @@ export class RedisJobStore implements IJobStoreV2 {
|
|||
from,
|
||||
expectActionId ?? '',
|
||||
expectCreatedAt != null ? String(expectCreatedAt) : '',
|
||||
notAfterMs != null ? String(notAfterMs) : '',
|
||||
String(ttl),
|
||||
terminal ? '1' : '0',
|
||||
String(
|
||||
|
|
@ -2591,6 +2597,9 @@ export class RedisJobStore implements IJobStoreV2 {
|
|||
...clearFields,
|
||||
...fields,
|
||||
);
|
||||
if (result === -1 && notAfterMs != null) {
|
||||
throw new JobStatusTransitionDeadlineError(notAfterMs);
|
||||
}
|
||||
if (returnDrainedSteers ? typeof result !== 'string' : result !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,12 @@ export {
|
|||
STEER_QUEUE_MAX_DEPTH,
|
||||
} from './interfaces/IJobStore';
|
||||
export { SteeringLifecycle, toPendingSteer } from './SteeringLifecycle';
|
||||
export {
|
||||
ApprovalLifecycle,
|
||||
PendingActionExpiredError,
|
||||
PENDING_ACTION_EXPIRED_CODE,
|
||||
} from './ApprovalLifecycle';
|
||||
export type { ApprovalLifecycleCallbacks, ApprovalPauseOptions } from './ApprovalLifecycle';
|
||||
export {
|
||||
assertJobStoreV2,
|
||||
getMissingJobStoreV2Methods,
|
||||
|
|
|
|||
|
|
@ -565,6 +565,17 @@ export const STEER_ENQUEUE_QUEUE_FULL = -2;
|
|||
* so idempotency evidence is never evicted inside its recovery window. */
|
||||
export const STEER_ENQUEUE_RECEIPT_FULL = -3;
|
||||
|
||||
/** The store rejected a status CAS because its atomic deadline had elapsed. */
|
||||
export class JobStatusTransitionDeadlineError extends Error {
|
||||
readonly notAfterMs: number;
|
||||
|
||||
constructor(notAfterMs: number) {
|
||||
super('The status transition deadline elapsed before the transition could commit');
|
||||
this.name = 'JobStatusTransitionDeadlineError';
|
||||
this.notAfterMs = notAfterMs;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Arguments for an atomic {@link IJobStore.transitionStatus} compare-and-set.
|
||||
*/
|
||||
|
|
@ -589,6 +600,12 @@ export interface JobStatusTransition {
|
|||
* the same stream ID.
|
||||
*/
|
||||
expectCreatedAt?: number;
|
||||
/**
|
||||
* Additional guard: reject the transition when the store's clock has reached
|
||||
* this absolute deadline. The comparison is part of the same atomic operation
|
||||
* as the status change, so queueing or storage latency cannot publish stale state.
|
||||
*/
|
||||
notAfterMs?: number;
|
||||
/** Extend all current steer receipts in the SAME atomic step as this
|
||||
* transition. Used by running→requires_action so no enqueue can land between
|
||||
* a pre-pause TTL pass and the status CAS. */
|
||||
|
|
|
|||
|
|
@ -903,7 +903,7 @@ export type ToolApprovalMode = z.infer<typeof toolApprovalModeSchema>;
|
|||
*
|
||||
* Shape mirrors `@librechat/agents`'s `ToolPolicyConfig` so the host can map it
|
||||
* directly into `createToolPolicyHook(config)`. The SDK does the evaluation
|
||||
* (`deny → bypass → allow → ask → dontAsk → fallthrough(ask)`); this config
|
||||
* (`deny → ask → allow → bypass → dontAsk → fallthrough(ask)`); this config
|
||||
* just describes the surface.
|
||||
*
|
||||
* Conventions:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue