LibreChat/api/server/controllers/agents/__tests__/resume.spec.js
Danny Avila 67b7b441b2
🛂 feat: Filter Model-Bound Content by Source (#14425)
* feat: introduce optional content protection seam

* feat: enforce source-aware content filters

* feat: complete source-aware content enforcement

* test: activate skill file-text fail-close fixtures

* fix: harden source-aware content filters

* fix: harden model-bound content filtering

* fix: preserve legacy filters and generated files

* fix: inspect shared scalar metadata

* test: align mocks with current dev dependencies

* feat: add persisted content filter safeguards

* feat: complete source-aware content filter enforcement

* fix: move resume content preflight into TypeScript

* fix: close content inspection edge cases

* fix: harden content protection boundaries

* fix: complete content protection safeguards

* test: align persisted memory filter coverage

* fix: reconcile content protection with current dev

* fix: reconcile content protection with latest dev

* fix: close content protection review gaps

* fix: enforce source-aware provider boundaries

* fix: preserve legacy PII preflight semantics

* test: stabilize stored branch preflight fixture

* fix: defer agent writes until protected model admission

* perf: harden source-aware model-bound filtering

* fix: canonicalize provider lineage before validation

* fix: satisfy model-bound callback type checks

* perf: Bound content protection filtering work

* fix: Bound submission array traversal

* fix: Stabilize bounded content snapshots

* fix: Scope model-bound traversal overflows

* fix: Preserve scoped content inspection

* fix: Accumulate aggregate traversal scopes

* fix: centralize content policy boundaries

* test: align deferred tool policy context

* test: align controller policy mocks

* style: normalize content protection imports

* fix: close content policy review gaps

* fix: narrow active skill policy config

* fix: address content protection review boundaries

* fix: retain exact provenance overflow sentinel

* fix: preserve literal and scoped provenance updates

* fix: narrow persisted edit provenance

* fix: isolate exact overflow attribution

* fix: centralize stored prompt protection

* fix: fail closed on incomplete transcript evidence

* fix: align canonical transcript routing

* refactor: centralize content policy preflights

* fix: isolate upload policy error typing

* style: sort policy preflight imports

* refactor: centralize content policy boundaries
2026-08-21 22:43:32 -04:00

3412 lines
125 KiB
JavaScript

/**
* Integration tests for the HITL resume controller (POST /agents/chat/resume).
*
* Drives the real `ResumeAgentController` end-to-end over supertest with the SDK
* run, durable checkpointer, Mongo, and concurrency cache mocked out. The pure
* decision/liveness helpers (`isPendingActionStale`, `mapToolApprovalResolutions`,
* `findUndecidedToolCalls`, `findDisallowedDecisions`, `buildAbortedResponseMetadata`,
* `sanitizeMessageForTransmit`) run for real via `requireActual`, so the test
* exercises the actual guard ladder and the pause -> approve -> resume -> finalize
* lifecycle rather than re-implemented stubs.
*
* Covers:
* - the authorization / staleness / agent-and-endpoint / actionId guard ladder
* - tool_approval validation (undecided, policy-disallowed decision)
* - ask_user_question answer requirement
* - concurrency gate (429) and the atomic single-winner claim (409)
* - the happy path: ACK, run reconstruction, resumeCompletion, finalize (save the
* now-finished response, emit done, complete job, prune checkpoint)
* - re-pause (no double finalize), abort-during-resume (no double finalize),
* and the resume-failure terminal path
*/
const express = require('express');
const request = require('supertest');
const { Tools, Constants, ResourceType, AgentCapabilities } = require('librechat-data-provider');
const USER_ID = 'user-1';
const TENANT_ID = 'tenant-1';
const AGENT_ID = 'agent-abc';
const CONVO_ID = 'convo-123';
const ACTION_ID = 'action-xyz';
const NEXT_ACTION_ID = 'action-next';
const RESPONSE_MSG_ID = 'resp-1';
const USER_MSG_ID = 'umsg-1';
const THREAD_PARENT_ID = 'thread-parent-1';
const mockLogger = {
debug: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
info: jest.fn(),
};
const mockJobStore = {
getJob: jest.fn(),
updateJob: jest.fn(),
};
const mockGenerationJobManager = {
getJob: jest.fn(),
getJobStore: jest.fn(() => mockJobStore),
getResumeState: jest.fn(),
setContentParts: jest.fn(),
/** Resume moves ownership and rebuilds armed interrupts from the durable queue. */
rearmQueuedPreempts: jest.fn().mockResolvedValue(0),
emitChunk: jest.fn(),
emitDone: jest.fn(),
emitError: jest.fn(),
claimTerminalJob: jest.fn(),
publishTerminalClaim: jest.fn(),
finishTerminalJob: jest.fn(),
completeJob: jest.fn(),
abortJob: jest.fn(),
beginProviderExecution: jest.fn(),
markProviderExecutionDrained: jest.fn(),
failPausePersistence: jest.fn(),
expireApproval: jest.fn(),
approvals: {
resolve: jest.fn(),
ownsPausePersistence: jest.fn(),
finishPausePersistence: jest.fn(),
},
};
const mockDeleteAgentCheckpoint = jest.fn();
const mockCaptureAgentCheckpointGeneration = jest.fn();
const mockDecrementPendingRequest = jest.fn();
const mockCheckAndIncrementPendingRequest = jest.fn();
const mockGetAgentCheckpointer = jest.fn();
const mockCheckpointGetTuple = jest.fn();
const mockSaveMessage = jest.fn();
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockGetFiles = jest.fn();
const mockGetAgent = jest.fn();
const mockGetActions = jest.fn();
const mockGetUserMemories = jest.fn();
const mockGetRoleByName = jest.fn();
const mockCheckAccess = jest.fn();
const mockCheckPermission = jest.fn();
const mockDecryptMetadata = jest.fn();
const mockDisposeClient = jest.fn();
const mockGetMCPRequestContext = jest.fn();
const mockCleanupMCPRequestContextForReq = jest.fn();
const mockRecordScheduleOutcome = jest.fn();
const mockIsScheduleLive = jest.fn();
const mockClaimScheduleResume = jest.fn();
const mockReleaseScheduleResumeClaim = jest.fn();
const mockFinalizeScheduleResumeClaim = jest.fn();
const mockReleaseScheduleResumeFence = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
logger: mockLogger,
}));
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
GenerationJobManager: mockGenerationJobManager,
captureAgentCheckpointGeneration: (...args) => mockCaptureAgentCheckpointGeneration(...args),
deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
isSteerPreemptSupported: jest.fn(() => true),
createMCPRuntimeRequestBody: ({ messageId, conversationId, parentMessageId }) => ({
messageId,
conversationId,
parentMessageId,
}),
getAgentCheckpointer: (...args) => mockGetAgentCheckpointer(...args),
checkAccess: (...args) => mockCheckAccess(...args),
}));
jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
getConvo: (...args) => mockGetConvo(...args),
getMessages: (...args) => mockGetMessages(...args),
getFiles: (...args) => mockGetFiles(...args),
getAgent: (...args) => mockGetAgent(...args),
getActions: (...args) => mockGetActions(...args),
getUserMemories: (...args) => mockGetUserMemories(...args),
getRoleByName: (...args) => mockGetRoleByName(...args),
}));
jest.mock('~/server/services/ActionService', () => ({
decryptMetadata: (...args) => mockDecryptMetadata(...args),
}));
jest.mock('~/server/services/PermissionService', () => ({
checkPermission: (...args) => mockCheckPermission(...args),
}));
jest.mock('~/server/services/Schedules', () => ({
recordScheduleOutcome: (...args) => mockRecordScheduleOutcome(...args),
claimScheduleResume: (...args) => mockClaimScheduleResume(...args),
releaseScheduleResumeClaim: (...args) => mockReleaseScheduleResumeClaim(...args),
finalizeScheduleResumeClaim: (...args) => mockFinalizeScheduleResumeClaim(...args),
releaseScheduleResumeFence: (...args) => mockReleaseScheduleResumeFence(...args),
isScheduleLive: (...args) => mockIsScheduleLive(...args),
}));
jest.mock('~/server/cleanup', () => ({
disposeClient: (...args) => mockDisposeClient(...args),
}));
jest.mock('~/server/services/MCPRequestContext', () => ({
getMCPRequestContext: (...args) => mockGetMCPRequestContext(...args),
cleanupMCPRequestContextForReq: (...args) => mockCleanupMCPRequestContextForReq(...args),
}));
// Import after mocks
const ResumeAgentController = require('~/server/controllers/agents/resume');
/** Drain the microtask + immediate queues so the post-ACK continuation settles. */
const flush = () => new Promise((resolve) => setImmediate(resolve));
/** A live, resolvable paused tool-approval job (single tool call `tc1`). */
function makeToolApprovalJob(overrides = {}) {
const metaOverrides = overrides.metadata ?? {};
const pendingOverrides = metaOverrides.pendingAction ?? {};
return {
status: 'requires_action',
createdAt: 1000,
abortController: new AbortController(),
...overrides,
metadata: {
userId: USER_ID,
tenantId: TENANT_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
responseMessageId: RESPONSE_MSG_ID,
sender: 'TestAgent',
iconURL: 'https://example.com/icon.png',
model: 'claude-test',
isTemporary: false,
userMessage: {
messageId: USER_MSG_ID,
parentMessageId: THREAD_PARENT_ID,
text: 'please run the tool',
},
...metaOverrides,
pendingAction: {
actionId: ACTION_ID,
expiresAt: Date.now() + 60_000,
payload: {
type: 'tool_approval',
action_requests: [{ tool_call_id: 'tc1' }],
review_configs: [{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'reject'] }],
},
...pendingOverrides,
},
},
};
}
/** A live, resolvable paused ask-user-question job. */
function makeAskUserJob(overrides = {}) {
const job = makeToolApprovalJob(overrides);
job.metadata.pendingAction.payload = {
type: 'ask_user_question',
question: 'What should I name the file?',
};
return job;
}
function makeAskUserBatchJob(overrides = {}) {
const job = makeToolApprovalJob(overrides);
job.metadata.pendingAction.payload = {
type: 'ask_user_question',
question: { question: 'Which environment?' },
questions: [
{ id: 'environment', question: 'Which environment?' },
{ id: 'window', question: 'Which time window?' },
],
tool_call_id: 'tc1',
};
return job;
}
/** A mock reconstructed client for the post-ACK path. */
function makeClient(overrides = {}) {
return {
sender: 'TestAgent',
contentParts: [{ type: 'text', text: 'resumed answer' }],
artifactPromises: [],
pendingApproval: false,
buildResponseMetadata: jest.fn(() => null),
resumeCompletion: jest.fn().mockResolvedValue(undefined),
...overrides,
};
}
function makeToolCallContent(overrides = {}) {
return {
type: 'tool_call',
tool_call: {
id: 'tc1',
name: 'lookup',
args: '{}',
...overrides,
},
};
}
describe('ResumeAgentController (POST /agents/chat/resume)', () => {
let app;
let mockInitializeClient;
let mockAddTitle;
let capturedInit;
let requestConfigOverrides;
let endpointAgent;
let settle;
let settled;
beforeEach(() => {
jest.clearAllMocks();
capturedInit = null;
requestConfigOverrides = {};
mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true });
mockDecrementPendingRequest.mockResolvedValue(undefined);
mockDeleteAgentCheckpoint.mockResolvedValue(undefined);
mockCaptureAgentCheckpointGeneration.mockResolvedValue({
threadId: CONVO_ID,
checkpointIds: ['checkpoint-old'],
});
mockCleanupMCPRequestContextForReq.mockResolvedValue(undefined);
mockSaveMessage.mockResolvedValue({});
mockGetConvo.mockResolvedValue(null);
mockGetMessages.mockResolvedValue([]);
mockGetFiles.mockResolvedValue([]);
mockGetAgent.mockResolvedValue(null);
mockGetActions.mockResolvedValue([]);
mockGetUserMemories.mockResolvedValue([]);
mockGetRoleByName.mockResolvedValue(null);
mockCheckAccess.mockResolvedValue(true);
mockCheckPermission.mockResolvedValue(true);
mockDecryptMetadata.mockImplementation(async (metadata) => metadata);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: { channel_values: { messages: [] } },
});
mockGetAgentCheckpointer.mockResolvedValue({ getTuple: mockCheckpointGetTuple });
mockJobStore.getJob.mockResolvedValue({
createdAt: 1000,
tokenUsage: null,
contextUsage: null,
});
mockJobStore.updateJob.mockResolvedValue(undefined);
mockGenerationJobManager.getResumeState.mockResolvedValue({ aggregatedContent: [] });
mockGenerationJobManager.emitDone.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);
mockGenerationJobManager.claimTerminalJob.mockResolvedValue({
streamId: CONVO_ID,
createdAt: 1000,
status: 'complete',
persistencePending: true,
drainedSteers: [],
});
mockGenerationJobManager.publishTerminalClaim.mockImplementation(
async (_claim, finalEvent) => ({
finalEvent: finalEvent ?? {
final: true,
reconcile: true,
reconcileReason: 'terminal_payload_missing',
terminalStatus: 'error',
},
persistenceFailed: finalEvent == null,
}),
);
mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(true);
mockGenerationJobManager.abortJob.mockResolvedValue({ success: true });
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);
mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);
mockRecordScheduleOutcome.mockResolvedValue(true);
mockIsScheduleLive.mockResolvedValue(true);
mockClaimScheduleResume.mockResolvedValue({
capacitySlot: 0,
claimToken: 'resume-token',
leaseBy: 'resume:resume-token',
});
mockReleaseScheduleResumeClaim.mockResolvedValue(true);
mockFinalizeScheduleResumeClaim.mockResolvedValue(true);
mockReleaseScheduleResumeFence.mockResolvedValue(undefined);
endpointAgent = {
_id: 'mongo-agent-abc',
id: AGENT_ID,
provider: 'openAI',
model: 'gpt-test',
instructions: 'Help the user.',
model_parameters: {},
tools: [],
edges: [],
};
// `decrementPendingRequest` runs in the controller's `finally` on every
// post-ACK path, so resolving on it signals the async continuation is done.
settled = new Promise((resolve) => {
settle = resolve;
});
mockDecrementPendingRequest.mockImplementation(async () => {
settle();
});
mockAddTitle = jest.fn().mockResolvedValue(undefined);
mockInitializeClient = jest.fn(async ({ req, checkpointNamespace, requestBody }) => {
// Capture the request state the controller seeds BEFORE reconstruction.
capturedInit = {
parentMessageId: req.body.parentMessageId,
files: req.body.files,
isTemporary: req.body.isTemporary,
conversationCreatedAt: req.conversationCreatedAt,
timezone: req.body.timezone,
checkpointNamespace,
requestBody,
};
return { client: makeClient(), userMCPAuthMap: { server1: { token: 't' } } };
});
app = express();
app.use(express.json());
app.use((req, _res, next) => {
req.user = { id: USER_ID, tenantId: TENANT_ID, role: 'USER' };
req.config = {
endpoints: { agents: { checkpointer: { type: 'mongo' } } },
interfaceConfig: {},
...requestConfigOverrides,
};
req.body.endpointOption = {
endpoint: 'agents',
agent_id: AGENT_ID,
model_parameters: {},
agent: Promise.resolve(endpointAgent),
};
next();
});
app.post('/api/agents/chat/resume', (req, res, next) =>
ResumeAgentController(req, res, next, mockInitializeClient, mockAddTitle),
);
});
const post = (body) => request(app).post('/api/agents/chat/resume').send(body);
const approveBody = (extra = {}) => ({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
decisions: [{ tool_call_id: 'tc1', decision: 'approve' }],
...extra,
});
describe('scheduled occurrence lifecycle', () => {
const scheduledFor = '2026-08-17T12:00:00.000Z';
const makeScheduledJob = () =>
makeToolApprovalJob({
metadata: {
scheduleId: 'schedule-1',
scheduledFor,
scheduleConfigRevision: 4,
checkpointNamespace: '1000',
},
});
it('stops and settles an occurrence that became inactive while awaiting approval', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockIsScheduleLive.mockResolvedValue(false);
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });
// `scheduledFor` identifies the OCCURRENCE: a later fire can redirect the
// schedule while this run sits paused, so the policy recheck validates the
// destination this run recorded rather than the schedule's current one.
expect(mockIsScheduleLive).toHaveBeenCalledWith('schedule-1', 4, {
automatic: true,
policy: true,
scheduledFor,
});
expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, {
expectedCreatedAt: 1000,
awaitProviderDrain: true,
});
expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
scheduleId: 'schedule-1',
scheduledFor,
streamId: CONVO_ID,
jobCreatedAt: 1000,
status: 'interrupted',
conversationId: CONVO_ID,
error: 'Schedule was disabled, changed, or deleted before approval',
});
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
undefined,
{ checkpointNamespace: '1000' },
);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('fails closed without settling or pruning when provider drain cannot be confirmed', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockIsScheduleLive.mockResolvedValue(false);
mockGenerationJobManager.abortJob.mockRejectedValue(new Error('drain timed out'));
const res = await post(approveBody());
expect(res.status).toBe(503);
expect(res.headers['retry-after']).toBe('1');
expect(res.body).toMatchObject({ code: 'SCHEDULE_STOP_UNCONFIRMED' });
expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
// `abortJob` reports `success: false` with a REASON on every failure path. Gating on
// the absence of a reason treated an unreached job and a replacement generation as
// confirmed stops, settling the occurrence and pruning a checkpoint on neither.
it.each([
['the job vanished before the abort landed', 'job_not_found'],
['a replacement generation owns the conversation', 'generation_replaced'],
['the generation is still live', 'job_still_active'],
])('refuses to settle or prune when %s', async (_label, failureReason) => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockIsScheduleLive.mockResolvedValue(false);
mockGenerationJobManager.abortJob.mockResolvedValue({ success: false, failureReason });
const res = await post(approveBody());
expect(res.status).toBe(503);
expect(res.headers['retry-after']).toBe('1');
expect(res.body).toMatchObject({ code: 'SCHEDULE_STOP_UNCONFIRMED' });
expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
// The exact regression: an abort that reported `success: false` and nothing else was
// read as a confirmed stop, so the occurrence was settled and its checkpoint pruned.
it('refuses to settle or prune on a bare unsuccessful abort with no reason', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockIsScheduleLive.mockResolvedValue(false);
mockGenerationJobManager.abortJob.mockResolvedValue({ success: false });
const res = await post(approveBody());
expect(res.status).toBe(503);
expect(res.body).toMatchObject({ code: 'SCHEDULE_STOP_UNCONFIRMED' });
expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
});
it('settles an occurrence whose generation was already terminal and drained', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockIsScheduleLive.mockResolvedValue(false);
// No transition was needed, but `awaitProviderDrain` still proved the provider
// segment can no longer persist — a stop, just not one this call made. Refusing
// here would 503 a permanently terminal generation on every retry.
mockGenerationJobManager.abortJob.mockResolvedValue({
success: false,
failureReason: 'already_settled',
});
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });
expect(mockRecordScheduleOutcome).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 'schedule-1', status: 'interrupted' }),
);
expect(mockDeleteAgentCheckpoint).toHaveBeenCalled();
});
it('refuses to settle the stale resume handoff on an unconfirmed stop', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockFinalizeScheduleResumeClaim.mockResolvedValue(false);
mockGenerationJobManager.abortJob.mockResolvedValue({
success: false,
failureReason: 'generation_replaced',
});
const res = await post(approveBody());
expect(res.status).toBe(503);
expect(res.headers['retry-after']).toBe('1');
expect(res.body).toMatchObject({ code: 'SCHEDULE_STOP_UNCONFIRMED' });
expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('records success after resumed persistence and before terminal publication', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
expect(mockClaimScheduleResume).toHaveBeenCalledWith('schedule-1', scheduledFor, {
expectedConfigRevision: 4,
automatic: true,
});
expect(mockFinalizeScheduleResumeClaim).toHaveBeenCalledWith(
'schedule-1',
'resume-token',
'resume:resume-token',
{ expectedConfigRevision: 4, automatic: true },
);
expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
scheduleId: 'schedule-1',
scheduledFor,
streamId: CONVO_ID,
jobCreatedAt: 1000,
status: 'success',
conversationId: CONVO_ID,
});
expect(mockRecordScheduleOutcome.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
);
});
it('keeps the approval paused when global scheduled-run capacity is full', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockClaimScheduleResume.mockResolvedValue({ conflict: 'capacity' });
const res = await post(approveBody());
expect(res.status).toBe(429);
expect(res.headers['retry-after']).toBe('1');
expect(res.body).toMatchObject({ code: 'SCHEDULE_CAPACITY' });
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
});
it('rolls back scheduled capacity when the approval CAS does not consume the action', async () => {
const job = makeScheduledJob();
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(mockReleaseScheduleResumeClaim).toHaveBeenCalledWith('schedule-1', scheduledFor, 0);
expect(mockReleaseScheduleResumeFence).toHaveBeenCalledWith(
'schedule-1',
'resume:resume-token',
);
});
it('stops before provider execution when an edit wins the final resume handoff', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockFinalizeScheduleResumeClaim.mockResolvedValue(false);
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' });
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalled();
expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, {
expectedCreatedAt: 1000,
awaitProviderDrain: true,
});
expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
scheduleId: 'schedule-1',
scheduledFor,
streamId: CONVO_ID,
jobCreatedAt: 1000,
status: 'interrupted',
conversationId: CONVO_ID,
error: 'Schedule was disabled, changed, or deleted before approval',
});
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('settles a scheduled continuation stopped during its resumed segment', async () => {
const job = makeScheduledJob();
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockInitializeClient.mockImplementation(async () => {
job.abortController.abort();
return { client: makeClient(), userMCPAuthMap: {} };
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
scheduleId: 'schedule-1',
scheduledFor,
streamId: CONVO_ID,
jobCreatedAt: 1000,
status: 'interrupted',
conversationId: CONVO_ID,
error: 'Scheduled run was stopped',
});
});
it('records an empty-preempt resumed segment as interrupted, not successful', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
run: {
getPreemptStats: () => ({ emptyBoundaries: 1 }),
getHaltReason: () => 'preempt_incomplete',
},
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ unfinished: true }),
expect.anything(),
);
expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
scheduleId: 'schedule-1',
scheduledFor,
streamId: CONVO_ID,
jobCreatedAt: 1000,
status: 'interrupted',
conversationId: CONVO_ID,
error: 'Scheduled run was interrupted before completion',
});
});
it('does not overwrite the terminal winner when failed-resume finalization loses its CAS', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockGenerationJobManager.completeJob.mockResolvedValue(false);
mockInitializeClient.mockRejectedValue(new Error('resume reconstruction failed'));
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();
expect(mockRecordScheduleOutcome).not.toHaveBeenCalled();
});
});
describe('temporal context restore', () => {
it('restores req.conversationCreatedAt from the convo before initializeClient', async () => {
// Temporal prompt vars must resolve against the paused anchor, not resume wall-clock.
mockGetConvo.mockResolvedValue({ createdAt: new Date('2020-01-02T03:04:05.000Z') });
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
expect(capturedInit.conversationCreatedAt).toBe('2020-01-02T03:04:05.000Z');
});
/**
* `approvals.resolve` atomically records capability and flips the job to
* running. Durable-arm reconstruction follows; ioredis queues rather than
* rejects during an outage, so an unbounded await there would strand the
* client after the action is already spent.
*/
it('answers the resume even when steering bookkeeping never settles', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
let releaseRearm;
mockGenerationJobManager.rearmQueuedPreempts.mockReturnValue(
new Promise((resolve) => {
releaseRearm = () => resolve(0);
}),
);
try {
const res = await post(approveBody());
expect(res.status).toBe(200);
expect(res.body.status).toBe('resuming');
} finally {
releaseRearm?.();
mockGenerationJobManager.rearmQueuedPreempts.mockResolvedValue(0);
}
}, 15000);
it('leaves conversationCreatedAt unset when the convo lookup yields nothing', async () => {
mockGetConvo.mockResolvedValue(null);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
expect(capturedInit.conversationCreatedAt).toBeUndefined();
});
});
describe('content policy preflight', () => {
it('preserves the default-off path without reading the durable checkpoint', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGetAgentCheckpointer).not.toHaveBeenCalled();
expect(mockInitializeClient).toHaveBeenCalledTimes(1);
});
it('does not read the checkpoint for a source unrelated to resume content', async () => {
requestConfigOverrides = {
filters: {
prompts: {
pii: {
fields: ['text'],
starterPatterns: ['sk_prefix'],
},
},
},
};
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGetAgentCheckpointer.mockRejectedValue(new Error('checkpoint unavailable'));
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGetAgentCheckpointer).not.toHaveBeenCalled();
expect(mockInitializeClient).toHaveBeenCalledTimes(1);
});
it('blocks the current saved agent instructions before consuming or acknowledging approval', async () => {
requestConfigOverrides = {
filters: {
agentInstructions: {
pii: {
fields: ['instructions'],
starterPatterns: ['sk_prefix'],
},
},
},
};
endpointAgent.instructions = 'Use sk-current-agent-secret when answering.';
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'agent_instruction',
field: 'instructions',
}),
);
expect(mockGetAgentCheckpointer).not.toHaveBeenCalled();
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockGetMCPRequestContext).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('blocks model parameters on a currently reachable nested agent before ACK', async () => {
requestConfigOverrides = {
filters: {
modelParameters: {
pii: {
fields: ['request_fields'],
starterPatterns: ['sk_prefix'],
},
},
},
};
endpointAgent.edges = [{ from: AGENT_ID, to: 'nested-agent' }];
mockGetAgent.mockResolvedValue({
_id: 'mongo-nested-agent',
id: 'nested-agent',
provider: 'openAI',
model: 'gpt-test',
model_parameters: { privateHeader: 'sk-nested-model-secret' },
tools: [],
edges: [],
});
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'model_parameter',
field: 'request_fields',
}),
);
expect(mockGetAgent).toHaveBeenCalledWith({ id: 'nested-agent' });
expect(mockCheckPermission).toHaveBeenCalledWith(
expect.objectContaining({
userId: USER_ID,
resourceType: ResourceType.AGENT,
resourceId: 'mongo-nested-agent',
}),
);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('blocks current action metadata before consuming approval', async () => {
requestConfigOverrides = {
filters: {
actionMetadata: {
pii: {
fields: ['privacy_policy_url'],
starterPatterns: ['sk_prefix'],
},
},
},
};
endpointAgent.tools = ['lookup_action_example'];
mockGetActions.mockResolvedValue([
{
action_id: 'action-1',
agent_id: AGENT_ID,
metadata: {
domain: 'example.test',
privacy_policy_url: 'sk-current-action-secret',
},
},
]);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(mockGetActions).toHaveBeenCalledWith({ agent_id: { $in: [AGENT_ID] } }, false);
expect(mockDecryptMetadata).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('blocks current model-bound memory before consuming approval', async () => {
requestConfigOverrides = {
memory: { disabled: false },
filters: {
memories: {
pii: {
fields: ['value'],
starterPatterns: ['sk_prefix'],
},
},
},
};
mockGetUserMemories.mockResolvedValue([
{ key: 'credential', value: 'sk-current-memory-secret' },
]);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(mockGetUserMemories).toHaveBeenCalledWith({
userId: USER_ID,
agentId: undefined,
});
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('claims and ACKs a safe current agent graph, action, and memory snapshot', async () => {
requestConfigOverrides = {
memory: { disabled: false },
endpoints: {
agents: {
checkpointer: { type: 'mongo' },
capabilities: [AgentCapabilities.memory],
},
},
filters: {
agentInstructions: {
pii: { fields: ['instructions'], starterPatterns: ['sk_prefix'] },
},
modelParameters: {
pii: { fields: ['request_fields'], starterPatterns: ['sk_prefix'] },
},
actionMetadata: {
pii: { fields: ['privacy_policy_url'], starterPatterns: ['sk_prefix'] },
},
memories: {
pii: { fields: ['value'], starterPatterns: ['sk_prefix'] },
},
},
};
endpointAgent.edges = [{ from: AGENT_ID, to: 'nested-agent' }];
endpointAgent.tools = ['lookup_action_example'];
mockGetAgent.mockResolvedValue({
_id: 'mongo-nested-agent',
id: 'nested-agent',
provider: 'openAI',
model: 'gpt-test',
instructions: 'Safe nested instructions.',
model_parameters: { privateHeader: 'safe-header' },
tools: [Tools.memory],
edges: [],
});
mockGetActions.mockResolvedValue([
{
action_id: 'action-1',
agent_id: AGENT_ID,
metadata: {
domain: 'example.test',
privacy_policy_url: 'https://example.test/privacy',
},
},
]);
mockGetUserMemories.mockResolvedValue([{ key: 'preference', value: 'Likes tea.' }]);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
expect(res.body).toEqual({
streamId: CONVO_ID,
conversationId: CONVO_ID,
status: 'resuming',
generationProtocolVersion: 1,
});
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerDrained: true,
providerExecutionId: expect.any(String),
}),
1000,
);
await settled;
await flush();
expect(mockInitializeClient).toHaveBeenCalledTimes(1);
});
it('blocks legacy checkpoint content before initializeClient', async () => {
requestConfigOverrides = {
filters: {
messages: {
pii: {
fields: ['text'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [{ role: 'user', content: 'sk-legacy-checkpoint-secret' }],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'message',
}),
);
expect(mockCheckpointGetTuple).toHaveBeenCalledWith({
configurable: {
thread_id: CONVO_ID,
checkpoint_ns: '',
},
});
expect(mockGetMessages).toHaveBeenCalledWith({
conversationId: CONVO_ID,
user: USER_ID,
});
expect(mockInitializeClient).not.toHaveBeenCalled();
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockGetMCPRequestContext).not.toHaveBeenCalled();
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
});
it('blocks checkpoint-only content from the active generation namespace', async () => {
requestConfigOverrides = {
filters: {
messages: {
pii: {
fields: ['text'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const job = makeToolApprovalJob({
metadata: { checkpointNamespace: 'generation-1000' },
});
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [{ role: 'user', content: 'sk-namespaced-checkpoint-secret' }],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'message',
}),
);
expect(mockCheckpointGetTuple).toHaveBeenCalledWith({
configurable: {
thread_id: CONVO_ID,
checkpoint_ns: '',
__librechat_checkpoint_ns: 'generation-1000',
},
});
expect(mockInitializeClient).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('keeps checkpoint tool arguments visible to files-only fail-close policy', async () => {
requestConfigOverrides = {
filters: {
files: {
pii: {
fields: ['content'],
starterPatterns: [],
uninspectable: 'block',
},
},
},
};
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [
{
role: 'assistant',
content: 'inspect the attachment',
tool_calls: [
{
name: 'inspect_file',
args: { file_id: 'checkpoint-tool-file' },
},
],
},
],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_uninspectable',
source: 'file',
field: 'content',
}),
);
expect(mockGetFiles).toHaveBeenCalledWith(
{ file_id: { $in: ['checkpoint-tool-file'] }, user: USER_ID, tenantId: TENANT_ID },
{},
{},
);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('keeps structured checkpoint text metadata visible to files-only policy', async () => {
requestConfigOverrides = {
filters: {
files: {
pii: {
fields: ['content'],
starterPatterns: [],
uninspectable: 'block',
},
},
},
};
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [
{
role: 'user',
content: [
{
type: 'text',
text: {
value: 'inspect the attachment',
annotations: [{ file_id: 'checkpoint-annotation-file' }],
},
},
],
},
],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_uninspectable',
source: 'file',
field: 'content',
}),
);
expect(mockGetFiles).toHaveBeenCalledWith(
{ file_id: { $in: ['checkpoint-annotation-file'] }, user: USER_ID, tenantId: TENANT_ID },
{},
{},
);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('blocks protected values nested in cyclic checkpoint tool arguments', async () => {
requestConfigOverrides = {
filters: {
toolArguments: {
pii: {
fields: ['arguments'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const cyclicArguments = { token: 'sk-cyclic-checkpoint-secret' };
cyclicArguments.self = cyclicArguments;
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [
{
role: 'assistant',
content: '',
tool_calls: [{ name: 'lookup', args: cyclicArguments }],
},
],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'tool_argument',
field: 'arguments',
}),
);
expect(mockInitializeClient).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('fails closed when selected checkpoint tool arguments cannot be fully traversed', async () => {
requestConfigOverrides = {
filters: {
toolArguments: {
pii: {
fields: ['arguments'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const deepArguments = {};
let current = deepArguments;
for (let depth = 0; depth < 30; depth++) {
current.nested = {};
current = current.nested;
}
Object.defineProperty(deepArguments, 'toJSON', {
value: () => {
throw new Error('cannot serialize');
},
});
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [
{
role: 'assistant',
content: '',
tool_calls: [{ name: 'lookup', args: deepArguments }],
},
],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_uninspectable',
source: 'tool_argument',
field: 'arguments',
}),
);
expect(mockInitializeClient).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('allows incomplete checkpoint arguments when only tool output is selected', async () => {
requestConfigOverrides = {
filters: {
toolArguments: {
pii: {
fields: ['output'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const deepArguments = {};
let current = deepArguments;
for (let depth = 0; depth < 30; depth++) {
current.nested = {};
current = current.nested;
}
Object.defineProperty(deepArguments, 'toJSON', {
value: () => {
throw new Error('cannot serialize');
},
});
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockCheckpointGetTuple.mockResolvedValue({
checkpoint: {
channel_values: {
messages: [
{
role: 'assistant',
content: '',
tool_calls: [{ name: 'lookup', args: deepArguments }],
},
],
},
},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockInitializeClient).toHaveBeenCalledTimes(1);
});
it('blocks legacy seed tool content before initializeClient', async () => {
requestConfigOverrides = {
filters: {
toolArguments: {
pii: {
fields: ['output'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [makeToolCallContent({ output: 'sk-legacy-seed-secret' })],
});
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'tool_argument',
field: 'output',
}),
);
expect(mockInitializeClient).not.toHaveBeenCalled();
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
});
it('blocks a user-authored respond decision before consuming the action', async () => {
requestConfigOverrides = {
filters: {
messages: {
pii: {
fields: ['decision_response'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'respond'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(
approveBody({
decisions: [
{
tool_call_id: 'tc1',
decision: 'respond',
responseText: 'sk-user-response-secret',
},
],
}),
);
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'message',
field: 'decision_response',
}),
);
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('blocks user-edited tool arguments before consuming the action', async () => {
requestConfigOverrides = {
filters: {
toolArguments: {
pii: {
fields: ['arguments'],
starterPatterns: ['sk_prefix'],
},
},
},
};
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'edit'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(
approveBody({
decisions: [
{
tool_call_id: 'tc1',
decision: 'edit',
editedArguments: { token: 'sk-user-edited-secret' },
},
],
}),
);
expect(res.status).toBe(400);
expect(res.body).toEqual(
expect.objectContaining({
error: 'content_filter_block',
source: 'tool_argument',
field: 'arguments',
}),
);
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
});
describe('MCP request-context lifecycle', () => {
it('pre-seeds the run-scoped MCP context before initializeClient and tears it down after', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled; // the controller's finally has run
// Seeded with a null `res` + cleanupOnResponse:false so the post-ACK tool load
// finds the existing store instead of getting undefined (res is already finished).
expect(mockGetMCPRequestContext).toHaveBeenCalledWith(expect.anything(), undefined, {
cleanupOnResponse: false,
});
// ...and seeded BEFORE the client (hence tool loading) is built.
expect(mockGetMCPRequestContext.mock.invocationCallOrder[0]).toBeLessThan(
mockInitializeClient.mock.invocationCallOrder[0],
);
// ...then torn down exactly once in the finally.
expect(mockCleanupMCPRequestContextForReq).toHaveBeenCalledTimes(1);
});
it('marks the exact resumed provider segment drained only after request cleanup', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
const resumePatch = mockGenerationJobManager.approvals.resolve.mock.calls[0][2];
expect(mockGenerationJobManager.beginProviderExecution).toHaveBeenCalledWith(
CONVO_ID,
1000,
resumePatch.providerExecutionId,
);
expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(
CONVO_ID,
1000,
resumePatch.providerExecutionId,
);
expect(mockCleanupMCPRequestContextForReq.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.markProviderExecutionDrained.mock.invocationCallOrder[0],
);
expect(
mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],
).toBeLessThan(mockInitializeClient.mock.invocationCallOrder[0]);
});
});
describe('request guards (rejected before claiming the action)', () => {
it('400 when conversationId is missing', async () => {
const res = await post({ actionId: ACTION_ID });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/conversationId is required/i);
expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
});
it('400 when conversationId is the "new" placeholder', async () => {
const res = await post({ conversationId: 'new', actionId: ACTION_ID });
expect(res.status).toBe(400);
expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
});
it('404 when there is no paused job for the conversation', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(null);
const res = await post(approveBody());
expect(res.status).toBe(404);
expect(res.body.error).toMatch(/no paused generation/i);
});
it('403 when the job belongs to another user', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { userId: 'someone-else' } }),
);
const res = await post(approveBody());
expect(res.status).toBe(403);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('fails closed when the stored job owner is missing', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { userId: undefined } }),
);
const res = await post(approveBody());
expect(res.status).toBe(403);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('403 on a tenant mismatch', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { tenantId: 'other-tenant' } }),
);
const res = await post(approveBody());
expect(res.status).toBe(403);
});
it('403 when the resume omits the paused agent_id', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ agent_id: undefined }));
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/different agent/i);
});
it('403 when the resume claims a different agent_id', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ agent_id: 'agent-other' }));
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/different agent/i);
});
it('403 when the resume claims a different endpoint', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ endpoint: 'bedrock' }));
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/different endpoint/i);
});
it('403 when the resume OMITS the paused endpoint (no fall-through to ephemeral)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ endpoint: undefined }));
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/different endpoint/i);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('409s a resume targeting a replaced generation before claiming the action', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ generationCreatedAt: 999 }));
expect(res.status).toBe(409);
expect(res.body).toEqual({ code: 'RUN_REPLACED', generationProtocolVersion: 1 });
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('409 when the job is not in requires_action (already terminal; no expire)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob({ status: 'running' }));
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body.error).toMatch(/no live pending action/i);
// Already resolved/terminal — nothing to expire.
expect(mockGenerationJobManager.expireApproval).not.toHaveBeenCalled();
});
it('409 AND drives expiry when the pending action has expired (stale)', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.expiresAt = Date.now() - 1_000;
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body.error).toMatch(/no live pending action/i);
// The stale action is expired NOW (expire CAS + terminal SSE) so an attached SSE
// client gets a terminal event instead of hanging until the periodic sweeper runs.
expect(mockGenerationJobManager.expireApproval).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
1000,
);
});
it('400 when actionId is missing', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ actionId: undefined }));
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/actionId is required/i);
});
it('409 when actionId targets a stale action', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody({ actionId: 'stale-action' }));
expect(res.status).toBe(409);
expect(res.body.error).toMatch(/stale action/i);
});
it('400 when a tool call is left undecided', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.action_requests = [
{ tool_call_id: 'tc1' },
{ tool_call_id: 'tc2' },
];
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'reject'] },
{ tool_call_id: 'tc2', allowed_decisions: ['approve', 'reject'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody()); // only decides tc1
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/must be decided/i);
expect(res.body.undecided).toEqual(['tc2']);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('403 when a decision is not permitted by the tool policy', async () => {
const job = makeToolApprovalJob();
// Policy restricts tc1 to reject only; an `approve` must be refused.
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['reject'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/not permitted/i);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('400 when an edit decision omits editedArguments', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'edit'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(
approveBody({ decisions: [{ tool_call_id: 'tc1', decision: 'edit' }] }),
);
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/editedArguments/i);
expect(res.body.incomplete).toEqual(['tc1']);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('400 when a respond decision omits responseText', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'respond'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(
approveBody({ decisions: [{ tool_call_id: 'tc1', decision: 'respond' }] }),
);
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/responseText/i);
});
it('accepts a complete edit decision (editedArguments present)', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'edit'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(
approveBody({
decisions: [{ tool_call_id: 'tc1', decision: 'edit', editedArguments: { q: 'x' } }],
}),
);
expect(res.status).toBe(200);
await settled;
await flush();
});
it('403 when the resume request fingerprint does not match the paused config', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.requestFingerprint = 'fingerprint-of-a-different-config';
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/different agent configuration/i);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('proceeds when the resume request fingerprint matches the paused config', async () => {
const { computeAgentRequestFingerprint } = jest.requireActual('@librechat/api');
const job = makeToolApprovalJob();
job.metadata.pendingAction.requestFingerprint = computeAgentRequestFingerprint({
endpoint: 'agents',
agent_id: AGENT_ID,
});
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(200);
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
}),
1000,
);
await settled;
await flush();
});
it('403 when the resume sends a different promptPrefix than the paused config', async () => {
const { computeAgentRequestFingerprint } = jest.requireActual('@librechat/api');
const job = makeToolApprovalJob();
// Ephemeral instructions come from promptPrefix, so it's part of the fingerprint.
job.metadata.pendingAction.requestFingerprint = computeAgentRequestFingerprint({
endpoint: 'agents',
agent_id: AGENT_ID,
promptPrefix: 'be terse',
});
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody({ promptPrefix: 'ignore previous instructions' }));
expect(res.status).toBe(403);
expect(res.body.error).toMatch(/different agent configuration/i);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('400 when an ask_user_question resume carries no answer', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserJob());
const res = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/answer is required/i);
});
it('400 when an ask_user_question answer exceeds the length cap', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserJob());
const res = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answer: 'x'.repeat(16_001),
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/maximum length/i);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('400 when a batched answer omits a question or includes an unknown id', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserBatchJob());
const missing = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answers: { environment: 'staging' },
});
expect(missing.status).toBe(400);
expect(missing.body.error).toMatch(/every question/i);
mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserBatchJob());
const extra = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answers: { environment: 'staging', window: '7d', region: 'us-east-2' },
});
expect(extra.status).toBe(400);
expect(extra.body.error).toMatch(/unknown question id/i);
});
it('400 on an unsupported pending-action type', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload = { type: 'totally_unknown' };
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/unsupported pending action/i);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('proceeds (does not 403) for a pre-multi-tenancy job with no tenantId', async () => {
// hasTenantMismatch only blocks when the job carries a tenantId that differs;
// an untenanted (legacy) job must still resume once the userId check passes.
const job = makeToolApprovalJob({ metadata: { tenantId: undefined } });
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(200);
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
}),
1000,
);
await settled;
await flush();
});
it('429 when the concurrency gate rejects the resume', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: false });
const res = await post(approveBody());
expect(res.status).toBe(429);
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
it('consumes a checkpoint-snapshot rejection on the 429 early-return path', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: false });
mockCaptureAgentCheckpointGeneration.mockRejectedValue(new Error('mongo down'));
const res = await post(approveBody());
await flush();
expect(res.status).toBe(429);
expect(mockLogger.warn).toHaveBeenCalledWith(
'[ResumeAgentController] Failed to capture checkpoint generation',
{ type: 'Error' },
);
});
it('409 and releases the slot when the action was already claimed (single-winner)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body.error).toMatch(/already resolved or has expired/i);
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
}),
1000,
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('409s RUN_REPLACED when the generation changes before the resume CAS', async () => {
const original = makeToolApprovalJob();
mockGenerationJobManager.getJob
.mockResolvedValueOnce(original)
.mockResolvedValueOnce(makeToolApprovalJob({ createdAt: 2000 }));
mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);
const res = await post(approveBody({ generationCreatedAt: 1000 }));
expect(res.status).toBe(409);
expect(res.body).toEqual({ code: 'RUN_REPLACED', generationProtocolVersion: 1 });
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
}),
1000,
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockInitializeClient).not.toHaveBeenCalled();
});
it('releases the slot when the claim itself throws (store error, not a leak)', async () => {
// The increment happens before the claim, which runs before the run's own
// try/finally — a store error here must still release the slot or a retry of the
// still-paused approval gets spuriously 429'd until the counter TTL expires.
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGenerationJobManager.approvals.resolve.mockRejectedValue(new Error('redis down'));
const res = await post(approveBody());
expect(res.status).toBe(500);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockInitializeClient).not.toHaveBeenCalled();
});
});
describe('happy path: approve -> reconstruct -> resume -> finalize', () => {
it('ACKs immediately and claims the action atomically with the submitted actionId', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const res = await post(approveBody());
expect(res.status).toBe(200);
expect(res.body).toEqual({
streamId: CONVO_ID,
conversationId: CONVO_ID,
status: 'resuming',
generationProtocolVersion: 1,
});
expect(mockCaptureAgentCheckpointGeneration).toHaveBeenCalledWith(CONVO_ID, {
type: 'mongo',
});
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
}),
1000,
);
expect(mockCaptureAgentCheckpointGeneration.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0],
);
await settled;
await flush();
});
it('preserves and exactly echoes a paused generation protocol v2 marker', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { generationProtocolVersion: 2 } }),
);
const res = await post(approveBody({ generationProtocolVersion: 2 }));
expect(res.status).toBe(200);
expect(res.headers['x-librechat-generation-protocol']).toBe('2');
expect(res.body).toEqual({
streamId: CONVO_ID,
conversationId: CONVO_ID,
status: 'resuming',
generationProtocolVersion: 2,
});
await settled;
await flush();
});
it('seeds the thread parent before reconstruction and maps the decision to the SDK', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
await post(approveBody());
await settled;
await flush();
// initializeAgent scopes thread files off req.body.parentMessageId, seeded
// from the paused user message's parent before initializeClient runs.
expect(capturedInit.parentMessageId).toBe(THREAD_PARENT_ID);
expect(capturedInit.requestBody).toEqual({
messageId: RESPONSE_MSG_ID,
conversationId: CONVO_ID,
parentMessageId: USER_MSG_ID,
});
expect(mockInitializeClient).toHaveBeenCalledTimes(1);
const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
expect(client.resumeCompletion).toHaveBeenCalledWith(
expect.objectContaining({
resumeValue: { tc1: { type: 'approve' } },
userMCPAuthMap: { server1: { token: 't' } },
}),
);
});
it('reuses the persisted MCP identity for edited and overridden turns', async () => {
const persistedMCPRequestBody = {
messageId: RESPONSE_MSG_ID,
conversationId: 'overridden-conversation',
parentMessageId: RESPONSE_MSG_ID,
};
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { mcpRequestBody: persistedMCPRequestBody } }),
);
await post(approveBody());
await settled;
await flush();
expect(capturedInit.requestBody).toBe(persistedMCPRequestBody);
});
it('reuses the persisted generation checkpoint namespace and keeps legacy fallback explicit', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { checkpointNamespace: 'generation-1000' } }),
);
await post(approveBody());
await settled;
await flush();
expect(capturedInit.checkpointNamespace).toBe('generation-1000');
expect(mockCaptureAgentCheckpointGeneration).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
undefined,
{ checkpointNamespace: 'generation-1000' },
);
});
it('passes persisted run steps into the rebuilt run for tool-result correlation', async () => {
const runSteps = [
{
id: 'step-approval',
index: 1,
type: 'tool_calls',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'tc1', name: 'approval_probe', args: '{}' }],
},
usage: null,
},
];
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [],
runSteps,
});
await post(approveBody());
await settled;
await flush();
const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
expect(client.resumeCompletion).toHaveBeenCalledWith(expect.objectContaining({ runSteps }));
});
it('reapplies retained ask answers before resuming a later tool approval', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({
metadata: {
resolvedAskUserQuestions: [
{
request: 'Which environment?',
output: 'staging',
toolCallId: 'ask-1',
},
],
},
}),
);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [
{
type: 'tool_call',
tool_call: { id: 'ask-1', name: 'ask_user_question', args: '' },
},
],
runSteps: [],
});
await post(approveBody());
await settled;
await flush();
const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
expect(client.resumeCompletion).toHaveBeenCalledWith(
expect.objectContaining({
seedContent: [
expect.objectContaining({
tool_call: expect.objectContaining({
id: 'ask-1',
args: JSON.stringify('Which environment?'),
output: 'staging',
progress: 1,
}),
}),
],
}),
);
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
}),
1000,
);
});
it('restores the paused user message files before reconstruction (execute-code files)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// The resume body carries no files; the controller must source them from the
// persisted user message so an approved code/read-file tool keeps its uploads.
mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'f1' }] }]);
await post(approveBody());
await settled;
await flush();
expect(capturedInit.files).toEqual([{ file_id: 'f1' }]);
});
it.each([
{ stored: true, supplied: false, expected: true },
{ stored: false, supplied: true, expected: false },
{ stored: undefined, supplied: true, expected: false },
])(
'restores authoritative isTemporary=$stored before reconstruction',
async ({ stored, supplied, expected }) => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { isTemporary: stored } }),
);
const res = await post(approveBody({ isTemporary: supplied }));
expect(res.status).toBe(200);
await settled;
expect(capturedInit.isTemporary).toBe(expected);
},
);
it('ignores client-supplied resume files, sourcing from the paused job (security)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// The paused turn's authoritative files (DB row); a crafted client tries to swap them.
mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'paused' }] }]);
await post(approveBody({ files: [{ file_id: 'attacker-supplied' }] }));
await settled;
await flush();
// The crafted client files must NOT reach initializeAgent — only the paused set.
expect(capturedInit.files).toEqual([{ file_id: 'paused' }]);
});
it('clears client-supplied resume files when the paused turn had none (security)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGetMessages.mockResolvedValue([{ files: [] }]); // the paused turn had no files
await post(approveBody({ files: [{ file_id: 'attacker-supplied' }] }));
await settled;
await flush();
expect(capturedInit.files).toEqual([]);
});
it('prefers job-metadata files over both the client body and the DB row', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({
metadata: {
userMessage: {
messageId: USER_MSG_ID,
parentMessageId: THREAD_PARENT_ID,
text: 'x',
files: [{ file_id: 'meta' }],
},
},
}),
);
mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'db' }] }]);
await post(approveBody({ files: [{ file_id: 'attacker-supplied' }] }));
await settled;
await flush();
expect(capturedInit.files).toEqual([{ file_id: 'meta' }]);
});
it('carries the restored files onto the final requestMessage (user bubble keeps attachments)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// job.metadata.userMessage is persisted without files; the final SSE must still
// carry the restored uploads or the user bubble loses its attachments on resume.
mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'f1', filename: 'a.pdf' }] }]);
await post(approveBody());
await settled;
await flush();
const [, finalEvent] = mockGenerationJobManager.publishTerminalClaim.mock.calls[0];
expect(finalEvent.requestMessage).toMatchObject({
messageId: USER_MSG_ID,
isCreatedByUser: true,
files: [{ file_id: 'f1', filename: 'a.pdf' }],
});
});
it('persists the response, claims terminal ownership, emits done, finishes, and prunes', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.objectContaining({ userId: USER_ID, isTemporary: false }),
expect.objectContaining({
messageId: RESPONSE_MSG_ID,
parentMessageId: USER_MSG_ID,
conversationId: CONVO_ID,
content: [{ type: 'text', text: 'resumed answer' }],
unfinished: false,
error: false,
isCreatedByUser: false,
user: USER_ID,
agent_id: AGENT_ID,
}),
expect.objectContaining({
context: 'api/server/controllers/agents/resume.js - resumed response end',
}),
);
// Assert the finalEvent STRUCTURE, not just the hardcoded `final: true` literal —
// a `final: true`-only check would still pass if the entire content / title /
// requestMessage build in finalizeResumedTurn were deleted.
const [terminalClaim, finalEvent] =
mockGenerationJobManager.publishTerminalClaim.mock.calls[0];
expect(terminalClaim).toMatchObject({ streamId: CONVO_ID, persistencePending: true });
expect(finalEvent).toMatchObject({
final: true,
conversation: { conversationId: CONVO_ID },
responseMessage: {
messageId: RESPONSE_MSG_ID,
content: [{ type: 'text', text: 'resumed answer' }],
unfinished: false,
},
requestMessage: { messageId: USER_MSG_ID, isCreatedByUser: true },
});
expect(typeof finalEvent.title).toBe('string');
expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
CONVO_ID,
'complete',
undefined,
1000,
{ persistencePending: true },
);
const claim = await mockGenerationJobManager.claimTerminalJob.mock.results[0].value;
expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(claim);
expect(mockGenerationJobManager.claimTerminalJob.mock.invocationCallOrder[0]).toBeLessThan(
mockSaveMessage.mock.invocationCallOrder[0],
);
expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
);
expect(
mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
).toBeLessThan(mockGenerationJobManager.finishTerminalJob.mock.invocationCallOrder[0]);
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
{ threadId: CONVO_ID, checkpointIds: ['checkpoint-old'] },
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
});
it('publishes reconciliation instead of a normal FINAL when the resumed response save returns no row', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGenerationJobManager.completeJob.mockResolvedValue(false);
mockSaveMessage.mockResolvedValue(undefined);
await post(approveBody());
await settled;
await flush();
const claim = await mockGenerationJobManager.claimTerminalJob.mock.results[0].value;
expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledTimes(1);
expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(claim, null);
expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(claim);
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
CONVO_ID,
'Resumed response could not be persisted before terminal publication',
1000,
);
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledTimes(1);
});
it('degrades a failed checkpoint snapshot to scoped no-op cleanup', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockCaptureAgentCheckpointGeneration.mockRejectedValue(new Error('mongo down'));
await post(approveBody());
await settled;
await flush();
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
{ threadId: CONVO_ID, checkpointIds: [] },
);
});
it('skips finalization when the job was replaced mid-resume', async () => {
// The paused job has createdAt 1000; a concurrent request reused this conversationId,
// so the live job now has a different createdAt — finalizing would clobber the newer
// turn's job. The finally still runs (slot release), so `settled` resolves.
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob({ createdAt: 1000 }));
mockJobStore.getJob.mockResolvedValue({
tokenUsage: null,
contextUsage: null,
createdAt: 2000,
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).not.toHaveBeenCalled();
expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockGenerationJobManager.claimTerminalJob).not.toHaveBeenCalled();
expect(mockGenerationJobManager.finishTerminalJob).not.toHaveBeenCalled();
});
it('does not write a completed response when a pause or abort wins the terminal claim', async () => {
const job = makeToolApprovalJob();
mockGenerationJobManager.getJob.mockResolvedValue(job);
let signalResumeBoundary;
const resumeBoundary = new Promise((resolve) => {
signalResumeBoundary = resolve;
});
let releaseResume;
const resumeGate = new Promise((resolve) => {
releaseResume = resolve;
});
const client = makeClient({
resumeCompletion: jest.fn(async () => {
signalResumeBoundary();
await resumeGate;
}),
});
mockInitializeClient.mockResolvedValue({ client, userMCPAuthMap: {} });
await post(approveBody());
await resumeBoundary;
// Cross-replica Stop wins durably while its abort pub/sub is delayed, so
// the local signal remains live and only the terminal CAS can fence this
// resumed completion's Mongo write.
mockGenerationJobManager.claimTerminalJob.mockResolvedValue(null);
releaseResume();
await settled;
await flush();
expect(job.abortController.signal.aborted).toBe(false);
expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
CONVO_ID,
'complete',
undefined,
1000,
{ persistencePending: true },
);
expect(mockSaveMessage).not.toHaveBeenCalled();
expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockGenerationJobManager.finishTerminalJob).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
});
it('finishes a winning terminal claim even when FINAL publication throws', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGenerationJobManager.publishTerminalClaim.mockRejectedValue(new Error('transport down'));
await post(approveBody());
await settled;
await flush();
const claim = await mockGenerationJobManager.claimTerminalJob.mock.results[0].value;
expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(claim);
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
CONVO_ID,
'transport down',
1000,
);
});
it('does not release the slot in the finally when the client already released it on pause', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// Simulate handleRunInterrupt having released the concurrency slot on a re-pause.
mockInitializeClient.mockResolvedValue({
client: makeClient({ pendingRequestReleased: true }),
userMCPAuthMap: {},
});
let disposed;
const disposedP = new Promise((resolve) => {
disposed = resolve;
});
mockDisposeClient.mockImplementation(() => disposed());
await post(approveBody());
await disposedP;
await flush();
// The finally must NOT double-release — handleRunInterrupt already did.
expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
});
it('persists tool artifacts produced by the resumed continuation as attachments', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const artifact = { type: 'image', file_id: 'img-1' };
// The lean resume path bypasses BaseClient.sendMessage's artifact await, so the
// controller must await client.artifactPromises itself (and drop null results).
mockInitializeClient.mockResolvedValue({
client: makeClient({
artifactPromises: [Promise.resolve(artifact), Promise.resolve(null)],
}),
userMCPAuthMap: {},
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ attachments: [artifact] }),
expect.anything(),
);
});
it('falls back to the aggregated store content when the live client content is empty', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// No live content on the rebuilt client → the saved response must use the
// pre-pause aggregated content from the store, not an empty array.
mockInitializeClient.mockResolvedValue({
client: makeClient({ contentParts: [] }),
userMCPAuthMap: {},
});
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [{ type: 'text', text: 'from-store' }],
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ content: [{ type: 'text', text: 'from-store' }] }),
expect.anything(),
);
});
it('strips malformed tool_call parts from the saved content', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
contentParts: [
{ type: 'text', text: 'kept' },
{ type: 'tool_call' }, // malformed: no tool_call payload — must be filtered
],
}),
userMCPAuthMap: {},
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ content: [{ type: 'text', text: 'kept' }] }),
expect.anything(),
);
});
it('merges previously persisted attachments with the resumed segment artifacts', async () => {
const priorArtifact = { type: 'image', file_id: 'prior-1' };
const newArtifact = { type: 'image', file_id: 'new-1' };
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// An earlier pause segment already saved an attachment on the response row.
mockGetMessages.mockResolvedValue([{ attachments: [priorArtifact] }]);
mockInitializeClient.mockResolvedValue({
client: makeClient({ artifactPromises: [Promise.resolve(newArtifact)] }),
userMCPAuthMap: {},
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ attachments: [priorArtifact, newArtifact] }),
expect.anything(),
);
});
it('persists the resumed run context calibration (contextMeta) onto the saved response', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const contextMeta = { calibrationRatio: 0.8 };
mockInitializeClient.mockResolvedValue({
client: makeClient({ contextMeta }),
userMCPAuthMap: {},
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ contextMeta }),
expect.anything(),
);
});
it('carries manualSkills/alwaysAppliedSkills onto the resumed requestMessage', async () => {
const job = makeToolApprovalJob();
job.metadata.userMessage.manualSkills = ['skill-a'];
job.metadata.userMessage.alwaysAppliedSkills = ['skill-b'];
mockGenerationJobManager.getJob.mockResolvedValue(job);
await post(approveBody());
await settled;
await flush();
const [, finalEvent] = mockGenerationJobManager.publishTerminalClaim.mock.calls[0];
expect(finalEvent.requestMessage).toMatchObject({
manualSkills: ['skill-a'],
alwaysAppliedSkills: ['skill-b'],
});
});
it('attaches client response metadata to the saved message when present', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
const contextUsage = { tokenCount: 1234 };
mockInitializeClient.mockResolvedValue({
client: makeClient({ buildResponseMetadata: jest.fn(() => ({ contextUsage })) }),
userMCPAuthMap: {},
});
await post(approveBody());
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ metadata: expect.objectContaining({ contextUsage }) }),
expect.anything(),
);
});
it('resumes an ask_user_question with the free-form answer', async () => {
const job = makeAskUserJob();
const olderAsk = {
type: 'tool_call',
tool_call: { id: 'older-ask', name: 'ask_user_question', args: '' },
};
const currentAsk = {
type: 'tool_call',
tool_call: { id: 'current-ask', name: 'ask_user_question', args: '' },
};
const answeredPart = makeToolCallContent({
id: 'current-ask',
name: 'ask_user_question',
output: 'call it report.pdf',
});
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [olderAsk, currentAsk],
});
mockInitializeClient.mockResolvedValue({
client: makeClient({
contentParts: [olderAsk, answeredPart, { type: 'text', text: 'Done' }],
}),
userMCPAuthMap: {},
});
const res = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answer: 'call it report.pdf',
});
expect(res.status).toBe(200);
await settled;
await flush();
const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
expect(client.resumeCompletion).toHaveBeenCalledWith(
expect.objectContaining({ resumeValue: { answer: 'call it report.pdf' } }),
);
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
resolvedAskUserQuestions: [
{
request: 'What should I name the file?',
output: 'call it report.pdf',
contentIndex: 1,
},
],
}),
1000,
);
expect(mockJobStore.updateJob).toHaveBeenCalledWith(
CONVO_ID,
{
userSubmittedMessageFieldPaths: [
{ path: '/content/1/tool_call/output', field: 'answer' },
],
},
1000,
);
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
userSubmittedMessageFieldPaths: [
{ path: '/content/1/tool_call/output', field: 'answer' },
],
}),
expect.anything(),
);
expect(mockSaveMessage.mock.calls[0][1]).not.toHaveProperty('userSubmittedPaths');
expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
CONVO_ID,
'complete',
undefined,
1000,
{ persistencePending: true },
);
});
it('retains an ID-less answer when earlier text exists but the ask part is missing', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserJob());
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [{ type: 'text', text: 'Let me check.' }],
});
const res = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answer: 'call it report.pdf',
});
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
resolvedAskUserQuestions: [
{
request: 'What should I name the file?',
output: 'call it report.pdf',
contentMissing: true,
},
],
}),
1000,
);
});
it('resumes a batched ask_user_question with answers keyed by question id', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserBatchJob());
const answers = { environment: 'staging', window: '7d' };
const output = JSON.stringify({ answers });
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [
makeToolCallContent({ id: 'tc1', name: 'ask_user_question', args: '', output: '' }),
],
});
mockInitializeClient.mockResolvedValue({
client: makeClient({
contentParts: [
makeToolCallContent({ id: 'tc1', name: 'ask_user_question', output }),
{ type: 'text', text: 'Done' },
],
}),
userMCPAuthMap: {},
});
const res = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answers,
});
expect(res.status).toBe(200);
await settled;
await flush();
const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
expect(client.resumeCompletion).toHaveBeenCalledWith(
expect.objectContaining({ resumeValue: { answers } }),
);
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
expect.objectContaining({
preemptCapable: true,
providerExecutionId: expect.any(String),
providerDrained: true,
resolvedAskUserQuestions: [
{
request: { questions: expect.any(Array) },
output,
toolCallId: 'tc1',
},
],
}),
1000,
);
const exactProvenance = [{ path: '/content/0/tool_call/output', field: 'answer' }];
expect(mockJobStore.updateJob).toHaveBeenCalledWith(
CONVO_ID,
{ userSubmittedMessageFieldPaths: exactProvenance },
1000,
);
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ userSubmittedMessageFieldPaths: exactProvenance }),
expect.anything(),
);
});
it('persists answer provenance for the payload tool_call_id in a multi-ask turn', async () => {
const job = makeAskUserJob();
job.metadata.pendingAction.payload.tool_call_id = 'ask-first';
const firstAsk = makeToolCallContent({
id: 'ask-first',
name: 'ask_user_question',
args: '',
output: '',
});
const secondAsk = makeToolCallContent({
id: 'ask-second',
name: 'ask_user_question',
args: '',
output: '',
});
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [firstAsk, secondAsk],
});
mockInitializeClient.mockResolvedValue({
client: makeClient({
contentParts: [
makeToolCallContent({
id: 'ask-first',
name: 'ask_user_question',
output: 'first answer',
}),
secondAsk,
],
}),
userMCPAuthMap: {},
});
const res = await post({
conversationId: CONVO_ID,
actionId: ACTION_ID,
agent_id: AGENT_ID,
endpoint: 'agents',
answer: 'first answer',
});
expect(res.status).toBe(200);
await settled;
await flush();
const exactProvenance = [{ path: '/content/0/tool_call/output', field: 'answer' }];
expect(mockJobStore.updateJob).toHaveBeenCalledWith(
CONVO_ID,
{ userSubmittedMessageFieldPaths: exactProvenance },
1000,
);
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ userSubmittedMessageFieldPaths: exactProvenance }),
expect.anything(),
);
});
it.each([
{
name: 'respond text',
resolution: { tool_call_id: 'tc1', decision: 'respond', responseText: 'human response' },
finalToolCall: { output: 'human response' },
expectedPath: '/content/0/tool_call/output',
expectedField: 'decision_response',
},
{
name: 'reject reason',
resolution: { tool_call_id: 'tc1', decision: 'reject', reason: 'human rejection' },
finalToolCall: { output: 'human rejection' },
expectedPath: '/content/0/tool_call/output',
expectedField: 'decision_reason',
},
{
name: 'edited arguments',
resolution: { tool_call_id: 'tc1', decision: 'edit', editedArguments: { q: 'human edit' } },
finalToolCall: { args: '{"q":"human edit"}' },
expectedPath: '/content/0/tool_call/args',
expectedField: undefined,
},
])(
'persists exact provenance for $name',
async ({ resolution, finalToolCall, expectedPath, expectedField }) => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.review_configs = [
{
tool_call_id: 'tc1',
allowed_decisions: ['approve', resolution.decision],
},
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [makeToolCallContent()],
});
mockInitializeClient.mockResolvedValue({
client: makeClient({
contentParts: [makeToolCallContent(finalToolCall), { type: 'text', text: 'Done' }],
}),
userMCPAuthMap: {},
});
await post(approveBody({ decisions: [resolution] }));
await settled;
await flush();
const expectedProvenance = expectedField
? {
userSubmittedMessageFieldPaths: [{ path: expectedPath, field: expectedField }],
}
: { userSubmittedPaths: [expectedPath] };
expect(mockJobStore.updateJob).toHaveBeenCalledWith(CONVO_ID, expectedProvenance, 1000);
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining(expectedProvenance),
expect.anything(),
);
if (expectedField) {
expect(mockSaveMessage.mock.calls[0][1]).not.toHaveProperty('userSubmittedPaths');
} else {
expect(mockSaveMessage.mock.calls[0][1]).not.toHaveProperty(
'userSubmittedMessageFieldPaths',
);
}
},
);
it('does not mark approve-only HITL content as user-submitted', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [makeToolCallContent()],
});
mockInitializeClient.mockResolvedValue({
client: makeClient({ contentParts: [makeToolCallContent({ output: 'tool result' })] }),
userMCPAuthMap: {},
});
await post(approveBody());
await settled;
await flush();
expect(mockJobStore.updateJob).not.toHaveBeenCalledWith(
CONVO_ID,
expect.objectContaining({ userSubmittedPaths: expect.anything() }),
);
expect(mockSaveMessage.mock.calls[0][1]).not.toHaveProperty('userSubmittedPaths');
});
it('generates a title for a first-turn pause before completing the stream', async () => {
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGetConvo.mockResolvedValue({ title: 'New Chat' });
await post(approveBody());
await settled;
await flush();
expect(mockAddTitle).toHaveBeenCalledTimes(1);
// Terminal ownership precedes title persistence and final stream publication.
expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
CONVO_ID,
'complete',
undefined,
1000,
{ persistencePending: true },
);
expect(mockGenerationJobManager.claimTerminalJob.mock.invocationCallOrder[0]).toBeLessThan(
mockAddTitle.mock.invocationCallOrder[0],
);
});
it('still finalizes the turn when first-turn title generation throws', async () => {
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGetConvo.mockResolvedValue({ title: 'New Chat' });
// Title generation is best-effort: a throw must not break the resumed turn.
mockAddTitle.mockRejectedValue(new Error('title service down'));
await post(approveBody());
await settled;
await flush();
expect(mockLogger.error).toHaveBeenCalled();
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(
expect.objectContaining({ streamId: CONVO_ID }),
expect.any(Object),
);
expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledTimes(1);
});
});
describe('non-finalizing outcomes', () => {
it('re-pause: does not finalize when the run pauses again', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({ pendingApproval: { actionId: NEXT_ACTION_ID } }),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
// It persists progress (unfinished) but must NOT finalize the turn.
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ unfinished: true }),
expect.anything(),
);
expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockGenerationJobManager.claimTerminalJob).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.ownsPausePersistence).toHaveBeenCalledWith(
CONVO_ID,
NEXT_ACTION_ID,
1000,
);
expect(mockGenerationJobManager.approvals.finishPausePersistence).toHaveBeenCalledWith(
CONVO_ID,
NEXT_ACTION_ID,
1000,
);
expect(mockGenerationJobManager.failPausePersistence).not.toHaveBeenCalled();
// The slot is still released and the client disposed.
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
});
it('re-pause: persists the segment content (unfinished) so an expiring re-pause keeps it', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: { actionId: NEXT_ACTION_ID },
contentParts: [{ type: 'text', text: 'streamed this segment' }],
artifactPromises: [],
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
content: [{ type: 'text', text: 'streamed this segment' }],
unfinished: true,
}),
expect.objectContaining({
context: 'api/server/controllers/agents/resume.js - re-pause progress persist',
}),
);
expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
});
it('re-pause: preserves HITL response provenance on the unfinished row', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['respond'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [makeToolCallContent()],
});
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: true,
contentParts: [makeToolCallContent({ output: 'human response' })],
}),
userMCPAuthMap: {},
});
await post(
approveBody({
decisions: [{ tool_call_id: 'tc1', decision: 'respond', responseText: 'human response' }],
}),
);
await settled;
await flush();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
unfinished: true,
userSubmittedMessageFieldPaths: [
{ path: '/content/0/tool_call/output', field: 'decision_response' },
],
}),
expect.anything(),
);
});
it('re-pause: persists artifacts produced before pausing again (unfinished)', async () => {
const artifact = { type: 'image', file_id: 'seg-1' };
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: { actionId: NEXT_ACTION_ID },
artifactPromises: [Promise.resolve(artifact)],
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
// No finalize, but the segment's artifact is persisted unfinished so the next
// resume's finalize can merge it (otherwise the fresh client drops it).
expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ attachments: [artifact], unfinished: true }),
expect.objectContaining({
context: 'api/server/controllers/agents/resume.js - re-pause progress persist',
}),
);
});
it('re-pause: holds the exact Stop/resume barrier until progress persistence settles', async () => {
let signalSaveStarted;
const saveStarted = new Promise((resolve) => {
signalSaveStarted = resolve;
});
let releaseSave;
const saveGate = new Promise((resolve) => {
releaseSave = resolve;
});
mockSaveMessage.mockImplementation(async () => {
signalSaveStarted();
await saveGate;
return {};
});
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: { actionId: NEXT_ACTION_ID },
contentParts: [{ type: 'text', text: 'must be durable before another action' }],
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await saveStarted;
expect(mockGenerationJobManager.approvals.ownsPausePersistence).toHaveBeenCalledWith(
CONVO_ID,
NEXT_ACTION_ID,
1000,
);
expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
releaseSave();
await settled;
await flush();
expect(mockGenerationJobManager.approvals.finishPausePersistence).toHaveBeenCalledWith(
CONVO_ID,
NEXT_ACTION_ID,
1000,
);
expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.approvals.finishPausePersistence.mock.invocationCallOrder[0],
);
});
it('re-pause: terminalizes instead of exposing the next action when persistence fails', async () => {
mockSaveMessage.mockRejectedValue(new Error('re-pause save failed'));
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: { actionId: NEXT_ACTION_ID },
contentParts: [{ type: 'text', text: 'progress that must not be lost' }],
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGenerationJobManager.failPausePersistence).toHaveBeenCalledWith(
CONVO_ID,
NEXT_ACTION_ID,
're-pause save failed',
1000,
);
expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
{ threadId: CONVO_ID, checkpointIds: ['checkpoint-old'] },
);
expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.failPausePersistence.mock.invocationCallOrder[0],
);
});
it('re-pause: contains checkpoint deletion failure after exact persistence terminalization', async () => {
const checkpointError = new Error('checkpoint delete unavailable');
mockSaveMessage.mockRejectedValue(new Error('re-pause save failed'));
mockDeleteAgentCheckpoint.mockRejectedValue(checkpointError);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: { actionId: NEXT_ACTION_ID },
contentParts: [{ type: 'text', text: 'progress that must not be lost' }],
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGenerationJobManager.failPausePersistence).toHaveBeenCalledTimes(1);
expect(mockLogger.error).toHaveBeenCalledWith(
'[ResumeAgentController] Failed to prune checkpoint after re-pause persistence failure',
{ type: 'Error' },
);
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
});
it('re-pause: leaves a replacement checkpoint untouched when exact persistence failure loses ownership', async () => {
mockSaveMessage.mockResolvedValue(undefined);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(false);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
pendingApproval: { actionId: NEXT_ACTION_ID },
contentParts: [{ type: 'text', text: 'stale segment progress' }],
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGenerationJobManager.failPausePersistence).toHaveBeenCalledWith(
CONVO_ID,
NEXT_ACTION_ID,
'Re-pause response progress could not be persisted',
1000,
);
expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
});
it('abort-during-resume: lets the abort route finalize, does not double-save', async () => {
const job = makeToolApprovalJob();
job.metadata.pendingAction.payload.review_configs = [
{ tool_call_id: 'tc1', allowed_decisions: ['respond'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockGenerationJobManager.getResumeState.mockResolvedValue({
aggregatedContent: [makeToolCallContent()],
});
mockInitializeClient.mockImplementation(async () => {
job.abortController.abort();
return { client: makeClient(), userMCPAuthMap: {} };
});
const res = await post(
approveBody({
decisions: [{ tool_call_id: 'tc1', decision: 'respond', responseText: 'human response' }],
}),
);
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockSaveMessage).not.toHaveBeenCalled();
expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockJobStore.updateJob).toHaveBeenCalledWith(
CONVO_ID,
{
userSubmittedMessageFieldPaths: [
{ path: '/content/0/tool_call/output', field: 'decision_response' },
],
},
1000,
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
});
it('resume failure delegates safe single-winner error publication and prunes the checkpoint', async () => {
const rawValue = 'PRIVATE-RESUME-PROVIDER-ECHO';
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
resumeCompletion: jest.fn().mockRejectedValue(new Error(`Provider echoed ${rawValue}`)),
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200); // already ACKed before the failure
await settled;
await flush();
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
CONVO_ID,
`Provider echoed ${rawValue}`,
1000,
);
expect(JSON.stringify(mockLogger.error.mock.calls)).not.toContain(rawValue);
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
{ threadId: CONVO_ID, checkpointIds: ['checkpoint-old'] },
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockSaveMessage).not.toHaveBeenCalled();
});
it('uses a normalized resume error when protection is active', async () => {
const rawValue = 'PRIVATE-RESUME-PROVIDER-ECHO';
requestConfigOverrides = { filters: { messages: { pii: {} } } };
const job = makeToolApprovalJob();
job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
mockGenerationJobManager.getJob.mockResolvedValue(job);
mockInitializeClient.mockResolvedValue({
client: makeClient({
resumeCompletion: jest.fn().mockRejectedValue(new Error(`Provider echoed ${rawValue}`)),
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
CONVO_ID,
'Resume failed',
1000,
);
expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain(
rawValue,
);
expect(JSON.stringify(mockLogger.error.mock.calls)).not.toContain(rawValue);
});
it('preserves resume error behavior for an unrelated-only source policy', async () => {
const rawValue = 'PROMPT-SOURCE-DOES-NOT-PARTICIPATE-IN-RESUME';
requestConfigOverrides = {
filters: {
prompts: {
pii: {
fields: ['text'],
starterPatterns: ['sk_prefix'],
},
},
},
};
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
resumeCompletion: jest.fn().mockRejectedValue(new Error(rawValue)),
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGetAgentCheckpointer).not.toHaveBeenCalled();
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, rawValue, 1000);
expect(JSON.stringify(mockLogger.error.mock.calls)).not.toContain(rawValue);
});
it('contains checkpoint deletion failure after winning ordinary resume-error finalization', async () => {
const checkpointError = new Error('checkpoint delete unavailable');
mockDeleteAgentCheckpoint.mockRejectedValue(checkpointError);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
}),
userMCPAuthMap: {},
});
const res = await post(approveBody());
expect(res.status).toBe(200);
await settled;
await flush();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, 'boom', 1000);
expect(mockLogger.error).toHaveBeenCalledWith(
'[ResumeAgentController] Failed to prune checkpoint after failed resume finalization',
{ type: 'Error' },
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
});
it('keeps a re-pause checkpoint when failed-resume terminal ownership is lost', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { checkpointNamespace: 'generation-1000' } }),
);
mockInitializeClient.mockResolvedValue({
client: makeClient({
resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
}),
userMCPAuthMap: {},
});
// A fresh interrupt moved the same epoch back to requires_action before
// the error path's terminal CAS. Its newly written checkpoint is live.
mockGenerationJobManager.completeJob.mockResolvedValue(false);
await post(approveBody());
await settled;
await flush();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, 'boom', 1000);
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
});
it('does not overwrite the job when guarded error finalization itself fails', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
}),
userMCPAuthMap: {},
});
// An unconditional fallback update could overwrite a pause or replacement
// that won while completeJob was failing.
mockGenerationJobManager.completeJob.mockRejectedValue(new Error('complete failed'));
await post(approveBody());
await settled;
await flush();
expect(mockJobStore.updateJob).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
});
});
});