LibreChat/api/server/controllers/agents/__tests__/resume.spec.js

/**
 * Integration tests for the HITL resume controller (POST /agents/chat/resume).
 *
 * Drives the real `ResumeAgentController` end-to-end over supertest with the SDK
 * run, durable checkpointer, Mongo, and concurrency cache mocked out. The pure
 * decision/liveness helpers (`isPendingActionStale`, `mapToolApprovalResolutions`,
 * `findUndecidedToolCalls`, `findDisallowedDecisions`, `buildAbortedResponseMetadata`,
 * `sanitizeMessageForTransmit`) run for real via `requireActual`, so the test
 * exercises the actual guard ladder and the pause -> approve -> resume -> finalize
 * lifecycle rather than re-implemented stubs.
 *
 * Covers:
 *  - the authorization / staleness / agent-and-endpoint / actionId guard ladder
 *  - tool_approval validation (undecided, policy-disallowed decision)
 *  - ask_user_question answer requirement
 *  - concurrency gate (429) and the atomic single-winner claim (409)
 *  - the happy path: ACK, run reconstruction, resumeCompletion, finalize (save the
 *    now-finished response, emit done, complete job, prune checkpoint)
 *  - re-pause (no double finalize), abort-during-resume (no double finalize),
 *    and the resume-failure terminal path
 */

const express = require('express');
const request = require('supertest');
const { Constants } = require('librechat-data-provider');

const USER_ID = 'user-1';
const TENANT_ID = 'tenant-1';
const AGENT_ID = 'agent-abc';
const CONVO_ID = 'convo-123';
const ACTION_ID = 'action-xyz';
const RESPONSE_MSG_ID = 'resp-1';
const USER_MSG_ID = 'umsg-1';
const THREAD_PARENT_ID = 'thread-parent-1';

const mockLogger = {
  debug: jest.fn(),
  warn: jest.fn(),
  error: jest.fn(),
  info: jest.fn(),
};

const mockJobStore = {
  getJob: jest.fn(),
  updateJob: jest.fn(),
};

const mockGenerationJobManager = {
  getJob: jest.fn(),
  getJobStore: jest.fn(() => mockJobStore),
  getResumeState: jest.fn(),
  setContentParts: jest.fn(),
  emitChunk: jest.fn(),
  emitDone: jest.fn(),
  emitError: jest.fn(),
  completeJob: jest.fn(),
  expireApproval: jest.fn(),
  approvals: { resolve: jest.fn() },
};

const mockDeleteAgentCheckpoint = jest.fn();
const mockDecrementPendingRequest = jest.fn();
const mockCheckAndIncrementPendingRequest = jest.fn();

const mockSaveMessage = jest.fn();
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockDisposeClient = jest.fn();
const mockGetMCPRequestContext = jest.fn();
const mockCleanupMCPRequestContextForReq = jest.fn();

jest.mock('@librechat/data-schemas', () => ({
  ...jest.requireActual('@librechat/data-schemas'),
  logger: mockLogger,
}));

jest.mock('@librechat/api', () => ({
  ...jest.requireActual('@librechat/api'),
  GenerationJobManager: mockGenerationJobManager,
  deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
  decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
  checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
}));

jest.mock('~/models', () => ({
  saveMessage: (...args) => mockSaveMessage(...args),
  getConvo: (...args) => mockGetConvo(...args),
  getMessages: (...args) => mockGetMessages(...args),
}));

jest.mock('~/server/cleanup', () => ({
  disposeClient: (...args) => mockDisposeClient(...args),
}));

jest.mock('~/server/services/MCPRequestContext', () => ({
  getMCPRequestContext: (...args) => mockGetMCPRequestContext(...args),
  cleanupMCPRequestContextForReq: (...args) => mockCleanupMCPRequestContextForReq(...args),
}));

// Import after mocks
const ResumeAgentController = require('~/server/controllers/agents/resume');

/** Drain the microtask + immediate queues so the post-ACK continuation settles. */
const flush = () => new Promise((resolve) => setImmediate(resolve));

/** A live, resolvable paused tool-approval job (single tool call `tc1`). */
function makeToolApprovalJob(overrides = {}) {
  const metaOverrides = overrides.metadata ?? {};
  const pendingOverrides = metaOverrides.pendingAction ?? {};
  return {
    status: 'requires_action',
    abortController: new AbortController(),
    ...overrides,
    metadata: {
      userId: USER_ID,
      tenantId: TENANT_ID,
      agent_id: AGENT_ID,
      endpoint: 'agents',
      responseMessageId: RESPONSE_MSG_ID,
      sender: 'TestAgent',
      iconURL: 'https://example.com/icon.png',
      model: 'claude-test',
      isTemporary: false,
      userMessage: {
        messageId: USER_MSG_ID,
        parentMessageId: THREAD_PARENT_ID,
        text: 'please run the tool',
      },
      ...metaOverrides,
      pendingAction: {
        actionId: ACTION_ID,
        expiresAt: Date.now() + 60_000,
        payload: {
          type: 'tool_approval',
          action_requests: [{ tool_call_id: 'tc1' }],
          review_configs: [{ tool_call_id: 'tc1', allowed_decisions: ['approve', 'reject'] }],
        },
        ...pendingOverrides,
      },
    },
  };
}

/** A live, resolvable paused ask-user-question job. */
function makeAskUserJob(overrides = {}) {
  const job = makeToolApprovalJob(overrides);
  job.metadata.pendingAction.payload = {
    type: 'ask_user_question',
    question: 'What should I name the file?',
  };
  return job;
}

/** A mock reconstructed client for the post-ACK path. */
function makeClient(overrides = {}) {
  return {
    sender: 'TestAgent',
    contentParts: [{ type: 'text', text: 'resumed answer' }],
    artifactPromises: [],
    pendingApproval: false,
    buildResponseMetadata: jest.fn(() => null),
    resumeCompletion: jest.fn().mockResolvedValue(undefined),
    ...overrides,
  };
}

describe('ResumeAgentController (POST /agents/chat/resume)', () => {
  let app;
  let mockInitializeClient;
  let mockAddTitle;
  let capturedInit;
  let settle;
  let settled;

  beforeEach(() => {
    jest.clearAllMocks();

    capturedInit = null;
    mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true });
    mockDecrementPendingRequest.mockResolvedValue(undefined);
    mockDeleteAgentCheckpoint.mockResolvedValue(undefined);
    mockCleanupMCPRequestContextForReq.mockResolvedValue(undefined);
    mockSaveMessage.mockResolvedValue(undefined);
    mockGetConvo.mockResolvedValue(null);
    mockGetMessages.mockResolvedValue([]);
    mockJobStore.getJob.mockResolvedValue({ tokenUsage: null, contextUsage: null });
    mockJobStore.updateJob.mockResolvedValue(undefined);
    mockGenerationJobManager.getResumeState.mockResolvedValue({ aggregatedContent: [] });
    mockGenerationJobManager.emitDone.mockResolvedValue(undefined);
    mockGenerationJobManager.emitError.mockResolvedValue(undefined);
    mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);
    mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
    mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);

    // `decrementPendingRequest` runs in the controller's `finally` on every
    // post-ACK path, so resolving on it signals the async continuation is done.
    settled = new Promise((resolve) => {
      settle = resolve;
    });
    mockDecrementPendingRequest.mockImplementation(async () => {
      settle();
    });

    mockAddTitle = jest.fn().mockResolvedValue(undefined);
    mockInitializeClient = jest.fn(async ({ req }) => {
      // Capture the request state the controller seeds BEFORE reconstruction.
      capturedInit = {
        parentMessageId: req.body.parentMessageId,
        files: req.body.files,
        conversationCreatedAt: req.conversationCreatedAt,
        timezone: req.body.timezone,
      };
      return { client: makeClient(), userMCPAuthMap: { server1: { token: 't' } } };
    });

    app = express();
    app.use(express.json());
    app.use((req, _res, next) => {
      req.user = { id: USER_ID, tenantId: TENANT_ID };
      req.config = {
        endpoints: { agents: { checkpointer: { type: 'mongo' } } },
        interfaceConfig: {},
      };
      next();
    });
    app.post('/api/agents/chat/resume', (req, res, next) =>
      ResumeAgentController(req, res, next, mockInitializeClient, mockAddTitle),
    );
  });

  const post = (body) => request(app).post('/api/agents/chat/resume').send(body);

  const approveBody = (extra = {}) => ({
    conversationId: CONVO_ID,
    actionId: ACTION_ID,
    agent_id: AGENT_ID,
    endpoint: 'agents',
    decisions: [{ tool_call_id: 'tc1', decision: 'approve' }],
    ...extra,
  });

  describe('temporal context restore', () => {
    it('restores req.conversationCreatedAt from the convo before initializeClient', async () => {
      // Temporal prompt vars must resolve against the paused anchor, not resume wall-clock.
      mockGetConvo.mockResolvedValue({ createdAt: new Date('2020-01-02T03:04:05.000Z') });
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled;
      expect(capturedInit.conversationCreatedAt).toBe('2020-01-02T03:04:05.000Z');
    });

    it('leaves conversationCreatedAt unset when the convo lookup yields nothing', async () => {
      mockGetConvo.mockResolvedValue(null);
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled;
      expect(capturedInit.conversationCreatedAt).toBeUndefined();
    });
  });

  describe('MCP request-context lifecycle', () => {
    it('pre-seeds the run-scoped MCP context before initializeClient and tears it down after', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled; // the controller's finally has run

      // Seeded with a null `res` + cleanupOnResponse:false so the post-ACK tool load
      // finds the existing store instead of getting undefined (res is already finished).
      expect(mockGetMCPRequestContext).toHaveBeenCalledWith(expect.anything(), undefined, {
        cleanupOnResponse: false,
      });
      // ...and seeded BEFORE the client (hence tool loading) is built.
      expect(mockGetMCPRequestContext.mock.invocationCallOrder[0]).toBeLessThan(
        mockInitializeClient.mock.invocationCallOrder[0],
      );
      // ...then torn down exactly once in the finally.
      expect(mockCleanupMCPRequestContextForReq).toHaveBeenCalledTimes(1);
    });
  });

  describe('request guards (rejected before claiming the action)', () => {
    it('400 when conversationId is missing', async () => {
      const res = await post({ actionId: ACTION_ID });
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/conversationId is required/i);
      expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
    });

    it('400 when conversationId is the "new" placeholder', async () => {
      const res = await post({ conversationId: 'new', actionId: ACTION_ID });
      expect(res.status).toBe(400);
      expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
    });

    it('404 when there is no paused job for the conversation', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(null);
      const res = await post(approveBody());
      expect(res.status).toBe(404);
      expect(res.body.error).toMatch(/no paused generation/i);
    });

    it('403 when the job belongs to another user', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(
        makeToolApprovalJob({ metadata: { userId: 'someone-else' } }),
      );
      const res = await post(approveBody());
      expect(res.status).toBe(403);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('403 on a tenant mismatch', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(
        makeToolApprovalJob({ metadata: { tenantId: 'other-tenant' } }),
      );
      const res = await post(approveBody());
      expect(res.status).toBe(403);
    });

    it('403 when the resume omits the paused agent_id', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody({ agent_id: undefined }));
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/different agent/i);
    });

    it('403 when the resume claims a different agent_id', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody({ agent_id: 'agent-other' }));
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/different agent/i);
    });

    it('403 when the resume claims a different endpoint', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody({ endpoint: 'bedrock' }));
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/different endpoint/i);
    });

    it('403 when the resume OMITS the paused endpoint (no fall-through to ephemeral)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody({ endpoint: undefined }));
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/different endpoint/i);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('409 when the job is not in requires_action (already terminal; no expire)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob({ status: 'running' }));
      const res = await post(approveBody());
      expect(res.status).toBe(409);
      expect(res.body.error).toMatch(/no live pending action/i);
      // Already resolved/terminal — nothing to expire.
      expect(mockGenerationJobManager.expireApproval).not.toHaveBeenCalled();
    });

    it('409 AND drives expiry when the pending action has expired (stale)', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.expiresAt = Date.now() - 1_000;
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody());
      expect(res.status).toBe(409);
      expect(res.body.error).toMatch(/no live pending action/i);
      // The stale action is expired NOW (expire CAS + terminal SSE) so an attached SSE
      // client gets a terminal event instead of hanging until the periodic sweeper runs.
      expect(mockGenerationJobManager.expireApproval).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
    });

    it('400 when actionId is missing', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody({ actionId: undefined }));
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/actionId is required/i);
    });

    it('409 when actionId targets a stale action', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody({ actionId: 'stale-action' }));
      expect(res.status).toBe(409);
      expect(res.body.error).toMatch(/stale action/i);
    });

    it('400 when a tool call is left undecided', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.payload.action_requests = [
        { tool_call_id: 'tc1' },
        { tool_call_id: 'tc2' },
      ];
      job.metadata.pendingAction.payload.review_configs = [
        { tool_call_id: 'tc1', allowed_decisions: ['approve', 'reject'] },
        { tool_call_id: 'tc2', allowed_decisions: ['approve', 'reject'] },
      ];
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody()); // only decides tc1
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/must be decided/i);
      expect(res.body.undecided).toEqual(['tc2']);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('403 when a decision is not permitted by the tool policy', async () => {
      const job = makeToolApprovalJob();
      // Policy restricts tc1 to reject only; an `approve` must be refused.
      job.metadata.pendingAction.payload.review_configs = [
        { tool_call_id: 'tc1', allowed_decisions: ['reject'] },
      ];
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody());
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/not permitted/i);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('400 when an edit decision omits editedArguments', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.payload.review_configs = [
        { tool_call_id: 'tc1', allowed_decisions: ['approve', 'edit'] },
      ];
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(
        approveBody({ decisions: [{ tool_call_id: 'tc1', decision: 'edit' }] }),
      );
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/editedArguments/i);
      expect(res.body.incomplete).toEqual(['tc1']);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('400 when a respond decision omits responseText', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.payload.review_configs = [
        { tool_call_id: 'tc1', allowed_decisions: ['approve', 'respond'] },
      ];
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(
        approveBody({ decisions: [{ tool_call_id: 'tc1', decision: 'respond' }] }),
      );
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/responseText/i);
    });

    it('accepts a complete edit decision (editedArguments present)', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.payload.review_configs = [
        { tool_call_id: 'tc1', allowed_decisions: ['approve', 'edit'] },
      ];
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(
        approveBody({
          decisions: [{ tool_call_id: 'tc1', decision: 'edit', editedArguments: { q: 'x' } }],
        }),
      );
      expect(res.status).toBe(200);
      await settled;
      await flush();
    });

    it('403 when the resume request fingerprint does not match the paused config', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.requestFingerprint = 'fingerprint-of-a-different-config';
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody());
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/different agent configuration/i);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('proceeds when the resume request fingerprint matches the paused config', async () => {
      const { computeAgentRequestFingerprint } = jest.requireActual('@librechat/api');
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.requestFingerprint = computeAgentRequestFingerprint({
        endpoint: 'agents',
        agent_id: AGENT_ID,
      });
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody());
      expect(res.status).toBe(200);
      expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
      await settled;
      await flush();
    });

    it('403 when the resume sends a different promptPrefix than the paused config', async () => {
      const { computeAgentRequestFingerprint } = jest.requireActual('@librechat/api');
      const job = makeToolApprovalJob();
      // Ephemeral instructions come from promptPrefix, so it's part of the fingerprint.
      job.metadata.pendingAction.requestFingerprint = computeAgentRequestFingerprint({
        endpoint: 'agents',
        agent_id: AGENT_ID,
        promptPrefix: 'be terse',
      });
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody({ promptPrefix: 'ignore previous instructions' }));
      expect(res.status).toBe(403);
      expect(res.body.error).toMatch(/different agent configuration/i);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('400 when an ask_user_question resume carries no answer', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserJob());
      const res = await post({
        conversationId: CONVO_ID,
        actionId: ACTION_ID,
        agent_id: AGENT_ID,
        endpoint: 'agents',
      });
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/answer is required/i);
    });

    it('400 on an unsupported pending-action type', async () => {
      const job = makeToolApprovalJob();
      job.metadata.pendingAction.payload = { type: 'totally_unknown' };
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody());
      expect(res.status).toBe(400);
      expect(res.body.error).toMatch(/unsupported pending action/i);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('proceeds (does not 403) for a pre-multi-tenancy job with no tenantId', async () => {
      // hasTenantMismatch only blocks when the job carries a tenantId that differs;
      // an untenanted (legacy) job must still resume once the userId check passes.
      const job = makeToolApprovalJob({ metadata: { tenantId: undefined } });
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      const res = await post(approveBody());
      expect(res.status).toBe(200);
      expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
      await settled;
      await flush();
    });

    it('429 when the concurrency gate rejects the resume', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: false });
      const res = await post(approveBody());
      expect(res.status).toBe(429);
      expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
    });

    it('409 and releases the slot when the action was already claimed (single-winner)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);
      const res = await post(approveBody());
      expect(res.status).toBe(409);
      expect(res.body.error).toMatch(/already resolved or has expired/i);
      expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
      expect(mockInitializeClient).not.toHaveBeenCalled();
    });

    it('releases the slot when the claim itself throws (store error, not a leak)', async () => {
      // The increment happens before the claim, which runs before the run's own
      // try/finally — a store error here must still release the slot or a retry of the
      // still-paused approval gets spuriously 429'd until the counter TTL expires.
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockGenerationJobManager.approvals.resolve.mockRejectedValue(new Error('redis down'));
      const res = await post(approveBody());
      expect(res.status).toBe(500);
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
      expect(mockInitializeClient).not.toHaveBeenCalled();
    });
  });

  describe('happy path: approve -> reconstruct -> resume -> finalize', () => {
    it('ACKs immediately and claims the action atomically with the submitted actionId', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const res = await post(approveBody());
      expect(res.status).toBe(200);
      expect(res.body).toEqual({
        streamId: CONVO_ID,
        conversationId: CONVO_ID,
        status: 'resuming',
      });
      expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
      await settled;
      await flush();
    });

    it('seeds the thread parent before reconstruction and maps the decision to the SDK', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      await post(approveBody());
      await settled;
      await flush();

      // initializeAgent scopes thread files off req.body.parentMessageId, seeded
      // from the paused user message's parent before initializeClient runs.
      expect(capturedInit.parentMessageId).toBe(THREAD_PARENT_ID);

      expect(mockInitializeClient).toHaveBeenCalledTimes(1);
      const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
      expect(client.resumeCompletion).toHaveBeenCalledWith(
        expect.objectContaining({
          resumeValue: { tc1: { type: 'approve' } },
          userMCPAuthMap: { server1: { token: 't' } },
        }),
      );
    });

    it('restores the paused user message files before reconstruction (execute-code files)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      // The resume body carries no files; the controller must source them from the
      // persisted user message so an approved code/read-file tool keeps its uploads.
      mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'f1' }] }]);

      await post(approveBody());
      await settled;
      await flush();

      expect(capturedInit.files).toEqual([{ file_id: 'f1' }]);
    });

    it('ignores client-supplied resume files, sourcing from the paused job (security)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      // The paused turn's authoritative files (DB row); a crafted client tries to swap them.
      mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'paused' }] }]);

      await post(approveBody({ files: [{ file_id: 'attacker-supplied' }] }));
      await settled;
      await flush();

      // The crafted client files must NOT reach initializeAgent — only the paused set.
      expect(capturedInit.files).toEqual([{ file_id: 'paused' }]);
    });

    it('clears client-supplied resume files when the paused turn had none (security)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockGetMessages.mockResolvedValue([{ files: [] }]); // the paused turn had no files

      await post(approveBody({ files: [{ file_id: 'attacker-supplied' }] }));
      await settled;
      await flush();

      expect(capturedInit.files).toEqual([]);
    });

    it('prefers job-metadata files over both the client body and the DB row', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(
        makeToolApprovalJob({
          metadata: {
            userMessage: {
              messageId: USER_MSG_ID,
              parentMessageId: THREAD_PARENT_ID,
              text: 'x',
              files: [{ file_id: 'meta' }],
            },
          },
        }),
      );
      mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'db' }] }]);

      await post(approveBody({ files: [{ file_id: 'attacker-supplied' }] }));
      await settled;
      await flush();

      expect(capturedInit.files).toEqual([{ file_id: 'meta' }]);
    });

    it('carries the restored files onto the final requestMessage (user bubble keeps attachments)', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      // job.metadata.userMessage is persisted without files; the final SSE must still
      // carry the restored uploads or the user bubble loses its attachments on resume.
      mockGetMessages.mockResolvedValue([{ files: [{ file_id: 'f1', filename: 'a.pdf' }] }]);

      await post(approveBody());
      await settled;
      await flush();

      const [, finalEvent] = mockGenerationJobManager.emitDone.mock.calls[0];
      expect(finalEvent.requestMessage).toMatchObject({
        messageId: USER_MSG_ID,
        isCreatedByUser: true,
        files: [{ file_id: 'f1', filename: 'a.pdf' }],
      });
    });

    it('persists the finished response, emits done, completes the job, and prunes the checkpoint', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.objectContaining({ userId: USER_ID, isTemporary: false }),
        expect.objectContaining({
          messageId: RESPONSE_MSG_ID,
          parentMessageId: USER_MSG_ID,
          conversationId: CONVO_ID,
          content: [{ type: 'text', text: 'resumed answer' }],
          unfinished: false,
          error: false,
          isCreatedByUser: false,
          user: USER_ID,
          agent_id: AGENT_ID,
        }),
        expect.objectContaining({
          context: 'api/server/controllers/agents/resume.js - resumed response end',
        }),
      );

      // Assert the finalEvent STRUCTURE, not just the hardcoded `final: true` literal —
      // a `final: true`-only check would still pass if the entire content / title /
      // requestMessage build in finalizeResumedTurn were deleted.
      const [doneStreamId, finalEvent] = mockGenerationJobManager.emitDone.mock.calls[0];
      expect(doneStreamId).toBe(CONVO_ID);
      expect(finalEvent).toMatchObject({
        final: true,
        conversation: { conversationId: CONVO_ID },
        responseMessage: {
          messageId: RESPONSE_MSG_ID,
          content: [{ type: 'text', text: 'resumed answer' }],
          unfinished: false,
        },
        requestMessage: { messageId: USER_MSG_ID, isCreatedByUser: true },
      });
      expect(typeof finalEvent.title).toBe('string');

      expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID);
      expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(CONVO_ID, { type: 'mongo' });
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
      expect(mockDisposeClient).toHaveBeenCalledTimes(1);
    });

    it('skips finalization (no save/emitDone/complete) when the job was replaced mid-resume', async () => {
      // The paused job has createdAt 1000; a concurrent request reused this conversationId,
      // so the live job now has a different createdAt — finalizing would clobber the newer
      // turn's job. The finally still runs (slot release), so `settled` resolves.
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob({ createdAt: 1000 }));
      mockJobStore.getJob.mockResolvedValue({
        tokenUsage: null,
        contextUsage: null,
        createdAt: 2000,
      });
      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).not.toHaveBeenCalled();
      expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
      expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
    });

    it('does not release the slot in the finally when the client already released it on pause', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      // Simulate handleRunInterrupt having released the concurrency slot on a re-pause.
      mockInitializeClient.mockResolvedValue({
        client: makeClient({ pendingRequestReleased: true }),
        userMCPAuthMap: {},
      });
      let disposed;
      const disposedP = new Promise((resolve) => {
        disposed = resolve;
      });
      mockDisposeClient.mockImplementation(() => disposed());

      await post(approveBody());
      await disposedP;
      await flush();

      // The finally must NOT double-release — handleRunInterrupt already did.
      expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
    });

    it('persists tool artifacts produced by the resumed continuation as attachments', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const artifact = { type: 'image', file_id: 'img-1' };
      // The lean resume path bypasses BaseClient.sendMessage's artifact await, so the
      // controller must await client.artifactPromises itself (and drop null results).
      mockInitializeClient.mockResolvedValue({
        client: makeClient({
          artifactPromises: [Promise.resolve(artifact), Promise.resolve(null)],
        }),
        userMCPAuthMap: {},
      });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ attachments: [artifact] }),
        expect.anything(),
      );
    });

    it('falls back to the aggregated store content when the live client content is empty', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      // No live content on the rebuilt client → the saved response must use the
      // pre-pause aggregated content from the store, not an empty array.
      mockInitializeClient.mockResolvedValue({
        client: makeClient({ contentParts: [] }),
        userMCPAuthMap: {},
      });
      mockGenerationJobManager.getResumeState.mockResolvedValue({
        aggregatedContent: [{ type: 'text', text: 'from-store' }],
      });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ content: [{ type: 'text', text: 'from-store' }] }),
        expect.anything(),
      );
    });

    it('strips malformed tool_call parts from the saved content', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockInitializeClient.mockResolvedValue({
        client: makeClient({
          contentParts: [
            { type: 'text', text: 'kept' },
            { type: 'tool_call' }, // malformed: no tool_call payload — must be filtered
          ],
        }),
        userMCPAuthMap: {},
      });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ content: [{ type: 'text', text: 'kept' }] }),
        expect.anything(),
      );
    });

    it('merges previously persisted attachments with the resumed segment artifacts', async () => {
      const priorArtifact = { type: 'image', file_id: 'prior-1' };
      const newArtifact = { type: 'image', file_id: 'new-1' };
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      // An earlier pause segment already saved an attachment on the response row.
      mockGetMessages.mockResolvedValue([{ attachments: [priorArtifact] }]);
      mockInitializeClient.mockResolvedValue({
        client: makeClient({ artifactPromises: [Promise.resolve(newArtifact)] }),
        userMCPAuthMap: {},
      });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ attachments: [priorArtifact, newArtifact] }),
        expect.anything(),
      );
    });

    it('persists the resumed run context calibration (contextMeta) onto the saved response', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const contextMeta = { calibrationRatio: 0.8 };
      mockInitializeClient.mockResolvedValue({
        client: makeClient({ contextMeta }),
        userMCPAuthMap: {},
      });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ contextMeta }),
        expect.anything(),
      );
    });

    it('carries manualSkills/alwaysAppliedSkills onto the resumed requestMessage', async () => {
      const job = makeToolApprovalJob();
      job.metadata.userMessage.manualSkills = ['skill-a'];
      job.metadata.userMessage.alwaysAppliedSkills = ['skill-b'];
      mockGenerationJobManager.getJob.mockResolvedValue(job);

      await post(approveBody());
      await settled;
      await flush();

      const [, finalEvent] = mockGenerationJobManager.emitDone.mock.calls[0];
      expect(finalEvent.requestMessage).toMatchObject({
        manualSkills: ['skill-a'],
        alwaysAppliedSkills: ['skill-b'],
      });
    });

    it('attaches client response metadata to the saved message when present', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      const contextUsage = { tokenCount: 1234 };
      mockInitializeClient.mockResolvedValue({
        client: makeClient({ buildResponseMetadata: jest.fn(() => ({ contextUsage })) }),
        userMCPAuthMap: {},
      });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ metadata: expect.objectContaining({ contextUsage }) }),
        expect.anything(),
      );
    });

    it('resumes an ask_user_question with the free-form answer', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeAskUserJob());
      const res = await post({
        conversationId: CONVO_ID,
        actionId: ACTION_ID,
        agent_id: AGENT_ID,
        endpoint: 'agents',
        answer: 'call it report.pdf',
      });
      expect(res.status).toBe(200);
      await settled;
      await flush();

      const client = await mockInitializeClient.mock.results[0].value.then((r) => r.client);
      expect(client.resumeCompletion).toHaveBeenCalledWith(
        expect.objectContaining({ resumeValue: { answer: 'call it report.pdf' } }),
      );
      expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID);
    });

    it('generates a title for a first-turn pause before completing the stream', async () => {
      const job = makeToolApprovalJob();
      job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      mockGetConvo.mockResolvedValue({ title: 'New Chat' });

      await post(approveBody());
      await settled;
      await flush();

      expect(mockAddTitle).toHaveBeenCalledTimes(1);
      // Title is emitted (and the job completed) — order matters but both must happen.
      expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID);
    });

    it('still finalizes the turn when first-turn title generation throws', async () => {
      const job = makeToolApprovalJob();
      job.metadata.userMessage.parentMessageId = Constants.NO_PARENT;
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      mockGetConvo.mockResolvedValue({ title: 'New Chat' });
      // Title generation is best-effort: a throw must not break the resumed turn.
      mockAddTitle.mockRejectedValue(new Error('title service down'));

      await post(approveBody());
      await settled;
      await flush();

      expect(mockLogger.error).toHaveBeenCalled();
      expect(mockSaveMessage).toHaveBeenCalledTimes(1);
      expect(mockGenerationJobManager.emitDone).toHaveBeenCalledWith(CONVO_ID, expect.any(Object));
      expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID);
    });
  });

  describe('non-finalizing outcomes', () => {
    it('re-pause: does not finalize when the run pauses again', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockInitializeClient.mockResolvedValue({
        client: makeClient({ pendingApproval: true }),
        userMCPAuthMap: {},
      });

      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled;
      await flush();

      // It persists progress (unfinished) but must NOT finalize the turn.
      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ unfinished: true }),
        expect.anything(),
      );
      expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
      expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
      expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
      // The slot is still released and the client disposed.
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
      expect(mockDisposeClient).toHaveBeenCalledTimes(1);
    });

    it('re-pause: persists the segment content (unfinished) so an expiring re-pause keeps it', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockInitializeClient.mockResolvedValue({
        client: makeClient({
          pendingApproval: true,
          contentParts: [{ type: 'text', text: 'streamed this segment' }],
          artifactPromises: [],
        }),
        userMCPAuthMap: {},
      });

      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled;
      await flush();

      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({
          content: [{ type: 'text', text: 'streamed this segment' }],
          unfinished: true,
        }),
        expect.objectContaining({
          context: 'api/server/controllers/agents/resume.js - re-pause progress persist',
        }),
      );
      expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
    });

    it('re-pause: persists artifacts produced before pausing again (unfinished)', async () => {
      const artifact = { type: 'image', file_id: 'seg-1' };
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockInitializeClient.mockResolvedValue({
        client: makeClient({
          pendingApproval: true,
          artifactPromises: [Promise.resolve(artifact)],
        }),
        userMCPAuthMap: {},
      });

      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled;
      await flush();

      // No finalize, but the segment's artifact is persisted unfinished so the next
      // resume's finalize can merge it (otherwise the fresh client drops it).
      expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
      expect(mockSaveMessage).toHaveBeenCalledWith(
        expect.anything(),
        expect.objectContaining({ attachments: [artifact], unfinished: true }),
        expect.objectContaining({
          context: 'api/server/controllers/agents/resume.js - re-pause progress persist',
        }),
      );
    });

    it('abort-during-resume: lets the abort route finalize, does not double-save', async () => {
      const job = makeToolApprovalJob();
      mockGenerationJobManager.getJob.mockResolvedValue(job);
      mockInitializeClient.mockImplementation(async () => {
        job.abortController.abort();
        return { client: makeClient(), userMCPAuthMap: {} };
      });

      const res = await post(approveBody());
      expect(res.status).toBe(200);
      await settled;
      await flush();

      expect(mockSaveMessage).not.toHaveBeenCalled();
      expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
      expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
    });

    it('resume failure: emits an error, finalizes the job, and prunes the checkpoint', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockInitializeClient.mockResolvedValue({
        client: makeClient({
          resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
        }),
        userMCPAuthMap: {},
      });

      const res = await post(approveBody());
      expect(res.status).toBe(200); // already ACKed before the failure
      await settled;
      await flush();

      expect(mockGenerationJobManager.emitError).toHaveBeenCalledWith(CONVO_ID, 'boom');
      expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, 'boom');
      expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(CONVO_ID, { type: 'mongo' });
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
      expect(mockSaveMessage).not.toHaveBeenCalled();
    });

    it('forces a terminal job state when completeJob also fails during a resume error', async () => {
      mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
      mockInitializeClient.mockResolvedValue({
        client: makeClient({
          resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
        }),
        userMCPAuthMap: {},
      });
      // The error path's completeJob also fails → last-resort updateJob must force a
      // terminal state so the job isn't orphaned in `running`.
      mockGenerationJobManager.completeJob.mockRejectedValue(new Error('complete failed'));

      await post(approveBody());
      await settled;
      await flush();

      expect(mockJobStore.updateJob).toHaveBeenCalledWith(
        CONVO_ID,
        expect.objectContaining({ status: 'error', error: 'Resume failed' }),
      );
      expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
    });
  });
});