LibreChat/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js

const { EventEmitter } = require('events');

const mockLogger = {
  debug: jest.fn(),
  warn: jest.fn(),
  error: jest.fn(),
  info: jest.fn(),
};

const mockGenerationJobManager = {
  createJob: jest.fn(),
  emitError: jest.fn(),
  completeJob: jest.fn(),
  getResumeState: jest.fn(),
  updateMetadata: jest.fn(),
  claimGeneration: jest.fn(),
  releaseGeneration: jest.fn(),
  hasJob: jest.fn(),
  steering: {
    closeAndDrain: jest.fn(),
    park: jest.fn(),
  },
};

const mockCheckAndIncrementPendingRequest = jest.fn();
const mockDecrementPendingRequest = jest.fn();
const mockGetViolationInfo = jest.fn(() => ({
  type: 'concurrent',
  limit: 2,
  pendingRequests: 3,
  score: 1,
}));
const mockFilterPersistableAbortContent = jest.fn((content) =>
  content.filter((part) => part?.type !== 'tool_call'),
);
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockSaveMessage = jest.fn();
const mockStartupTelemetry = {
  mark: jest.fn(),
  setStreamId: jest.fn(),
  recordGenerationEvent: jest.fn(),
  end: jest.fn(),
};
const mockGetAgentStartupTelemetry = jest.fn(() => mockStartupTelemetry);
const mockAcceptAgentStartupTelemetry = jest.fn();
let mockMCPContexts = new WeakMap();

const mockCreateMCPRequestContext = jest.fn(() => ({
  connections: new Map(),
  pending: new Map(),
  cleanupStarted: false,
  cleanupOnResponse: false,
  responseCleanupAttached: false,
}));
const mockGetMCPRequestContext = jest.fn((req) => {
  if (!req) {
    return undefined;
  }

  let context = mockMCPContexts.get(req);
  if (!context) {
    context = mockCreateMCPRequestContext();
    mockMCPContexts.set(req, context);
  }

  return context.cleanupStarted ? undefined : context;
});
const mockCleanupMCPRequestContext = jest.fn(async (context) => {
  if (!context || context.cleanupStarted) {
    return;
  }

  context.cleanupStarted = true;
  const connections = new Set(context.connections.values());
  const settled = await Promise.allSettled(context.pending.values());
  for (const result of settled) {
    if (result.status === 'fulfilled' && result.value) {
      connections.add(result.value);
    }
  }

  await Promise.allSettled(Array.from(connections).map((connection) => connection.disconnect?.()));
  context.connections.clear();
  context.pending.clear();
});
const mockCleanupMCPRequestContextForReq = jest.fn(async (req) => {
  const context = mockMCPContexts.get(req);
  if (!context) {
    return;
  }

  try {
    await mockCleanupMCPRequestContext(context);
  } finally {
    mockMCPContexts.delete(req);
  }
});

jest.mock('@librechat/data-schemas', () => ({
  logger: mockLogger,
}));

jest.mock('@librechat/api', () => ({
  sendEvent: jest.fn(),
  /** Recorded onto the job so the steer route can honour the OWNING replica's
   *  seal capability rather than its own probe. */
  isSteerPreemptSupported: jest.fn(() => true),
  getViolationInfo: (...args) => mockGetViolationInfo(...args),
  buildMessageFiles: jest.fn(() => []),
  resolveTitleTiming: jest.fn(() => 'immediate'),
  resolveConversationAnchor: jest.requireActual('@librechat/api').resolveConversationAnchor,
  GenerationJobManager: mockGenerationJobManager,
  getReferencedQuotes: jest.fn((quotes) => {
    if (!Array.isArray(quotes)) {
      return null;
    }
    const normalized = quotes
      .filter((quote) => typeof quote === 'string' && quote.trim().length > 0)
      .map((quote) => quote.trim());
    return normalized.length > 0 ? normalized : null;
  }),
  cleanupMCPRequestContext: (...args) => mockCleanupMCPRequestContext(...args),
  createMCPRequestContext: (...args) => mockCreateMCPRequestContext(...args),
  getMCPRequestContext: (...args) => mockGetMCPRequestContext(...args),
  filterPersistableAbortContent: (...args) => mockFilterPersistableAbortContent(...args),
  cleanupMCPRequestContextForReq: (...args) => mockCleanupMCPRequestContextForReq(...args),
  decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
  sanitizeMessageForTransmit: jest.fn((message) => message),
  checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
  getAgentStartupTelemetry: (...args) => mockGetAgentStartupTelemetry(...args),
  acceptAgentStartupTelemetry: (...args) => mockAcceptAgentStartupTelemetry(...args),
  isUnpersistedPreliminaryParent: async ({
    userId,
    conversationId,
    parentMessageId,
    getMessages,
  }) => {
    if (typeof parentMessageId !== 'string' || !parentMessageId.endsWith('_')) {
      return false;
    }

    const filter = { user: userId, messageId: parentMessageId };
    if (conversationId && conversationId !== 'new') {
      filter.conversationId = conversationId;
    }

    const messages = await getMessages(filter, '_id');
    return messages.length === 0;
  },
}));

jest.mock('~/server/cleanup', () => ({
  disposeClient: jest.fn(),
  clientRegistry: null,
  requestDataMap: {
    set: jest.fn(),
  },
}));

jest.mock('~/server/middleware', () => ({
  handleAbortError: jest.fn(() => Promise.resolve()),
}));

jest.mock('~/cache', () => ({
  logViolation: jest.fn(),
}));

jest.mock('~/models', () => ({
  saveMessage: (...args) => mockSaveMessage(...args),
  getMessages: (...args) => mockGetMessages(...args),
  getConvo: (...args) => mockGetConvo(...args),
}));

const AgentController = require('../request');
const { disposeClient: mockDisposeClient } = require('~/server/cleanup');
const { getMCPRequestContext } = require('~/server/services/MCPRequestContext');

function createResumableResponse() {
  const res = new EventEmitter();
  res.headersSent = false;
  res.writableEnded = false;
  res.finished = false;
  res.destroyed = false;
  res.json = jest.fn(() => {
    res.headersSent = true;
    res.writableEnded = true;
    res.finished = true;
    res.emit('finish');
    return res;
  });
  res.status = jest.fn(() => res);
  return res;
}

function nextTick() {
  return new Promise((resolve) => setImmediate(resolve));
}

describe('ResumableAgentController resume metadata', () => {
  beforeEach(() => {
    jest.clearAllMocks();
    mockMCPContexts = new WeakMap();
    mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true });
    mockDecrementPendingRequest.mockResolvedValue(undefined);
    mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' });
    mockGetMessages.mockResolvedValue([]);
    mockGenerationJobManager.createJob.mockResolvedValue({
      createdAt: 1000,
      readyPromise: Promise.resolve(),
      abortController: new AbortController(),
      emitter: { on: jest.fn() },
    });
    mockGenerationJobManager.getResumeState.mockResolvedValue(null);
    mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
    mockGenerationJobManager.emitError.mockResolvedValue(undefined);
    mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);
    mockGenerationJobManager.hasJob.mockResolvedValue(true);
    mockGenerationJobManager.steering.closeAndDrain.mockResolvedValue([]);
    mockGenerationJobManager.steering.park.mockResolvedValue(undefined);
    mockSaveMessage.mockResolvedValue({});
  });

  it('rejects an underscore-suffixed parent that is not persisted', async () => {
    const conversationId = 'conversation-123';
    const initializeClient = jest.fn();
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Follow up too early.',
        messageId: 'follow-up-user',
        parentMessageId: 'pending-response_',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          modelOptions: { model: 'gpt-3.5-turbo' },
        },
      },
      config: {},
    };
    const res = {
      json: jest.fn(),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGetMessages).toHaveBeenCalledWith(
      { user: 'user-123', messageId: 'pending-response_', conversationId },
      '_id',
    );
    expect(res.status).toHaveBeenCalledWith(409);
    expect(res.json).toHaveBeenCalledWith(
      expect.objectContaining({
        error: expect.stringContaining('selected parent response is still being saved'),
      }),
    );
    expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
    expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
    expect(initializeClient).not.toHaveBeenCalled();
  });

  it('allows an underscore-suffixed parent when it is already persisted', async () => {
    const conversationId = 'conversation-123';
    mockGetMessages.mockResolvedValue([{ _id: 'persisted-parent' }]);
    const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Follow up to persisted underscore id.',
        messageId: 'follow-up-user',
        parentMessageId: 'persisted-response_',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          modelOptions: { model: 'gpt-3.5-turbo' },
        },
      },
      config: {},
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGetMessages).toHaveBeenCalledWith(
      { user: 'user-123', messageId: 'persisted-response_', conversationId },
      '_id',
    );
    expect(res.status).not.toHaveBeenCalledWith(409);
    expect(mockCheckAndIncrementPendingRequest).toHaveBeenCalledWith('user-123');
    expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
      conversationId,
      'user-123',
      conversationId,
      expect.objectContaining({
        startupTelemetry: mockStartupTelemetry,
        initialMetadata: expect.objectContaining({
          conversationId,
          endpoint: 'agents',
        }),
      }),
    );
  });

  it('creates the job with the in-flight turn before MCP initialization can emit OAuth', async () => {
    const conversationId = 'conversation-123';
    const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Check Google Workspace availability.',
        messageId: 'follow-up-user',
        parentMessageId: 'original-response',
        conversationId,
        isTemporary: true,
        endpointOption: {
          endpoint: 'agents',
          iconURL: 'https://example.com/spec-icon.png',
          modelOptions: { model: 'gpt-3.5-turbo' },
        },
      },
      config: {},
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
      conversationId,
      'user-123',
      conversationId,
      {
        startupTelemetry: mockStartupTelemetry,
        initialMetadata: {
          conversationId,
          endpoint: 'agents',
          iconURL: 'https://example.com/spec-icon.png',
          model: 'gpt-3.5-turbo',
          /** The OWNING replica's seal capability, read by the steer route. */
          preemptCapable: true,
          agent_id: undefined,
          isTemporary: true,
          responseMessageId: 'follow-up-user_',
          userMessage: {
            messageId: 'follow-up-user',
            parentMessageId: 'original-response',
            conversationId,
            text: 'Check Google Workspace availability.',
          },
        },
      },
    );
    expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(
      initializeClient.mock.invocationCallOrder[0],
    );
    expect(mockGenerationJobManager.updateMetadata).not.toHaveBeenCalled();
    const startupMilestones = mockStartupTelemetry.mark.mock.calls.map(([milestone]) => milestone);
    expect(startupMilestones.slice(0, 2)).toEqual(['request_admitted', 'job_created']);
    expect(new Set(startupMilestones.slice(2))).toEqual(
      new Set(['conversation_resolved', 'metadata_persisted']),
    );
    expect(mockAcceptAgentStartupTelemetry).toHaveBeenCalledWith(req, conversationId);
    expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
  });

  it('prefetches conversation state before admission and joins it with job metadata', async () => {
    let resolveConversation;
    let signalMetadataStarted;
    const conversationPromise = new Promise((resolve) => {
      resolveConversation = resolve;
    });
    const metadataStarted = new Promise((resolve) => {
      signalMetadataStarted = resolve;
    });
    mockGetConvo.mockReturnValue(conversationPromise);
    mockGenerationJobManager.createJob.mockImplementation(() => {
      signalMetadataStarted();
      return Promise.resolve({
        createdAt: 1000,
        readyPromise: Promise.resolve(),
        abortController: new AbortController(),
        emitter: { on: jest.fn() },
      });
    });
    const initializeClient = jest.fn().mockRejectedValue(new Error('stop after startup reads'));
    const conversationId = 'conversation-123';
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Run independent startup work together.',
        messageId: 'user-message',
        parentMessageId: 'parent-message',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          modelOptions: { model: 'gpt-4.1' },
        },
      },
      config: {},
    };
    const res = createResumableResponse();

    const controllerPromise = AgentController(req, res, jest.fn(), initializeClient, null);
    expect(mockGetConvo).toHaveBeenCalledWith('user-123', conversationId);
    await metadataStarted;
    await nextTick();

    expect(mockGetConvo.mock.invocationCallOrder[0]).toBeLessThan(
      mockCheckAndIncrementPendingRequest.mock.invocationCallOrder[0],
    );
    expect(res.json).toHaveBeenCalledWith({
      streamId: conversationId,
      conversationId,
      status: 'started',
    });
    expect(initializeClient).not.toHaveBeenCalled();

    resolveConversation({ createdAt: '2026-06-07T00:00:00.000Z' });
    await controllerPromise;

    expect(initializeClient).toHaveBeenCalledTimes(1);
  });

  it('keeps request-scoped MCP connections until resumable initialization finishes', async () => {
    const conversationId = 'conversation-123';
    const disconnect = jest.fn().mockResolvedValue(undefined);
    const initializeClient = jest.fn(async ({ req, res }) => {
      const context = getMCPRequestContext(req, res);
      context.connections.set('mcp-server', { disconnect });

      await nextTick();
      expect(disconnect).not.toHaveBeenCalled();

      throw new Error('stop after request-scoped MCP connection');
    });
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Use a BODY-scoped MCP server.',
        messageId: 'user-message',
        parentMessageId: 'parent-message',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          modelOptions: { model: 'gpt-4.1' },
        },
      },
      config: {},
    };
    const res = createResumableResponse();

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(res.json).toHaveBeenCalledWith({
      streamId: conversationId,
      conversationId,
      status: 'started',
    });
    expect(disconnect).toHaveBeenCalledTimes(1);
    expect(disconnect.mock.invocationCallOrder[0]).toBeLessThan(
      mockDecrementPendingRequest.mock.invocationCallOrder[0],
    );
  });

  it('stores model spec icon fallbacks and agent ids in early resume metadata', async () => {
    const conversationId = 'conversation-123';
    const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Use the resume spec.',
        messageId: 'follow-up-user',
        parentMessageId: 'original-response',
        conversationId,
        isTemporary: true,
        endpointOption: {
          endpoint: 'agents',
          spec: 'agent-spec',
          agent_id: 'agent_resume_spec',
          model_parameters: { model: 'gpt-4.1' },
        },
      },
      config: {
        modelSpecs: {
          list: [
            {
              name: 'agent-spec',
              preset: {
                endpoint: 'openAI',
                iconURL: 'https://example.com/preset-icon.png',
              },
            },
          ],
        },
      },
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
      conversationId,
      'user-123',
      conversationId,
      expect.objectContaining({
        initialMetadata: expect.objectContaining({
          iconURL: 'https://example.com/preset-icon.png',
          model: 'agent_resume_spec',
          agent_id: 'agent_resume_spec',
          isTemporary: true,
        }),
      }),
    );
  });

  it('falls back to the model spec preset endpoint when no icon URL is configured', async () => {
    const conversationId = 'conversation-123';
    const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Use the endpoint icon.',
        messageId: 'follow-up-user',
        parentMessageId: 'original-response',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          spec: 'endpoint-icon-spec',
          model_parameters: { model: 'gpt-4.1' },
        },
      },
      config: {
        modelSpecs: {
          list: [
            {
              name: 'endpoint-icon-spec',
              preset: {
                endpoint: 'anthropic',
              },
            },
          ],
        },
      },
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
      conversationId,
      'user-123',
      conversationId,
      expect.objectContaining({
        initialMetadata: expect.objectContaining({
          iconURL: 'anthropic',
          model: 'gpt-4.1',
        }),
      }),
    );
  });

  it('filters OAuth prompts before saving partial responses on disconnect', async () => {
    const conversationId = 'conversation-123';
    let allSubscribersLeftHandler;
    mockGenerationJobManager.createJob.mockResolvedValue({
      createdAt: 1000,
      readyPromise: Promise.resolve(),
      abortController: new AbortController(),
      emitter: {
        on: jest.fn((event, handler) => {
          if (event === 'allSubscribersLeft') {
            allSubscribersLeftHandler = handler;
          }
        }),
      },
    });
    mockGenerationJobManager.getResumeState.mockResolvedValue({
      conversationId,
      responseMessageId: 'response-message',
      iconURL: 'https://example.com/spec-icon.png',
      model: 'gpt-4.1',
      userMessage: {
        messageId: 'user-message',
        parentMessageId: 'parent-message',
        conversationId,
        text: 'Use Google Workspace',
      },
    });

    const initializeClient = jest.fn().mockRejectedValue(new Error('stop after setup'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Use Google Workspace',
        messageId: 'user-message',
        parentMessageId: 'parent-message',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          iconURL: 'https://example.com/fallback-icon.png',
          modelOptions: { model: 'gpt-3.5-turbo' },
        },
      },
      config: {},
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);
    expect(allSubscribersLeftHandler).toEqual(expect.any(Function));

    const oauthPart = {
      type: 'tool_call',
      tool_call: {
        name: 'oauth_mcp_Google-Workspace',
        auth: 'https://auth.example.com/oauth',
      },
    };
    const textPart = { type: 'text', text: 'Partial response...' };

    await allSubscribersLeftHandler([oauthPart, textPart]);

    expect(mockFilterPersistableAbortContent).toHaveBeenCalledWith([oauthPart, textPart]);
    expect(mockSaveMessage).toHaveBeenCalledWith(
      expect.objectContaining({ userId: 'user-123' }),
      expect.objectContaining({
        content: [textPart],
        iconURL: 'https://example.com/spec-icon.png',
        model: 'gpt-4.1',
        messageId: 'response-message',
        parentMessageId: 'user-message',
      }),
      expect.any(Object),
    );
  });

  it('uses model spec and agent fallbacks when saving partial responses on disconnect', async () => {
    const conversationId = 'conversation-123';
    let allSubscribersLeftHandler;
    mockGenerationJobManager.createJob.mockResolvedValue({
      createdAt: 1000,
      readyPromise: Promise.resolve(),
      abortController: new AbortController(),
      emitter: {
        on: jest.fn((event, handler) => {
          if (event === 'allSubscribersLeft') {
            allSubscribersLeftHandler = handler;
          }
        }),
      },
    });
    mockGenerationJobManager.getResumeState.mockResolvedValue({
      conversationId,
      responseMessageId: 'response-message',
      userMessage: {
        messageId: 'user-message',
        parentMessageId: 'parent-message',
        conversationId,
        text: 'Use fallback metadata',
      },
    });

    const initializeClient = jest.fn().mockRejectedValue(new Error('stop after setup'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Use fallback metadata',
        messageId: 'user-message',
        parentMessageId: 'parent-message',
        conversationId,
        endpointOption: {
          endpoint: 'agents',
          spec: 'agent-spec',
          agent_id: 'agent_resume_spec',
          model_parameters: { model: 'gpt-4.1' },
        },
      },
      config: {
        modelSpecs: {
          list: [
            {
              name: 'agent-spec',
              preset: {
                endpoint: 'openAI',
                iconURL: 'https://example.com/preset-icon.png',
              },
            },
          ],
        },
      },
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);
    expect(allSubscribersLeftHandler).toEqual(expect.any(Function));

    const textPart = { type: 'text', text: 'Partial response...' };
    await allSubscribersLeftHandler([textPart]);

    expect(mockSaveMessage).toHaveBeenCalledWith(
      expect.objectContaining({ userId: 'user-123' }),
      expect.objectContaining({
        content: [textPart],
        iconURL: 'https://example.com/preset-icon.png',
        model: 'agent_resume_spec',
        messageId: 'response-message',
        parentMessageId: 'user-message',
      }),
      expect.any(Object),
    );
  });

  it('dedups a retried start-generation request to the original stream', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({
      claimed: false,
      existing: { streamId: 'orig-stream', conversationId: 'orig-convo' },
    });
    mockGenerationJobManager.hasJob.mockResolvedValue(true);
    const initializeClient = jest.fn();
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Retried after a lost response.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(res.json).toHaveBeenCalledWith({
      streamId: 'orig-stream',
      conversationId: 'orig-convo',
      status: 'resumed',
    });
    expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
    expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
    expect(initializeClient).not.toHaveBeenCalled();
    expect(mockStartupTelemetry.end).toHaveBeenCalledWith('deduplicated');
  });

  it('resumes when the job is missing but the claim is old (original completed and was cleaned up)', async () => {
    // An old claim with no job means the original already ran and was cleaned up; the deduped
    // response must attach (client 404 handler refetches) rather than loop on readiness.
    mockGenerationJobManager.claimGeneration.mockResolvedValue({
      claimed: false,
      existing: {
        streamId: 'orig-stream',
        conversationId: 'orig-convo',
        claimedAt: Date.now() - 60000,
      },
    });
    mockGenerationJobManager.hasJob.mockResolvedValue(false);
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Retry after a fast, already-cleaned-up generation.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };

    await AgentController(req, res, jest.fn(), jest.fn(), null);

    expect(res.json).toHaveBeenCalledWith({
      streamId: 'orig-stream',
      conversationId: 'orig-convo',
      status: 'resumed',
    });
    expect(res.status).not.toHaveBeenCalledWith(503);
    expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
  });

  it('returns 503 SERVER_NOT_READY when a fresh claim still has no job (winner is between claim and createJob)', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({
      claimed: false,
      existing: {
        streamId: 'orig-stream',
        conversationId: 'orig-convo',
        claimedAt: Date.now(),
      },
    });
    mockGenerationJobManager.hasJob.mockResolvedValue(false);
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Concurrent duplicate before the winner wrote its job.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };

    await AgentController(req, res, jest.fn(), jest.fn(), null);

    expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
    expect(res.status).toHaveBeenCalledWith(503);
    expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
    expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
  });

  it('never starts a second generation when the job lookup fails for a confirmed duplicate', async () => {
    // A store hiccup while checking an existing claim must not fail open into createJob.
    mockGenerationJobManager.claimGeneration.mockResolvedValue({
      claimed: false,
      existing: { streamId: 'orig-stream', conversationId: 'orig-convo', claimedAt: Date.now() },
    });
    mockGenerationJobManager.hasJob.mockRejectedValue(new Error('redis down'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Duplicate during a Redis hiccup.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };

    await AgentController(req, res, jest.fn(), jest.fn(), null);

    expect(res.status).toHaveBeenCalledWith(503);
    expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
    expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
  });

  it('does not finalize an unscoped generation when job creation rejects before returning', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    mockGenerationJobManager.createJob.mockRejectedValue(new Error('create failed before return'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Fail before receiving a job epoch.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = createResumableResponse();

    await AgentController(req, res, jest.fn(), jest.fn(), null);

    expect(res.status).toHaveBeenCalledWith(500);
    expect(res.json).toHaveBeenCalledWith({ error: 'create failed before return' });
    expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
    expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
    expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
    expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
  });

  it('finalizes the failed job before releasing the idempotency claim', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Start fails after the initial JSON.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = createResumableResponse();

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
      'conversation-123',
      expect.any(String),
      1000,
    );
    expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
    // completeJob must finalize the failed job BEFORE the claim is released, or a racing
    // retry could win the key, createJob the same streamId, and be aborted by this completeJob.
    expect(mockGenerationJobManager.completeJob.mock.invocationCallOrder[0]).toBeLessThan(
      mockGenerationJobManager.releaseGeneration.mock.invocationCallOrder[0],
    );
  });

  it('still releases the claim and pending slot when completeJob fails during init-error cleanup', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    mockGenerationJobManager.completeJob.mockRejectedValue(new Error('store hiccup'));
    const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Start fails while the store is degraded.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = createResumableResponse();

    await AgentController(req, res, jest.fn(), initializeClient, null);

    // A completeJob rejection must not wedge the retry behind the claim or leak the slot.
    expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
    expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
  });

  it('still finalizes and releases when streaming the initialization error fails', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
    const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Start fails while Redis publish is degraded.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = createResumableResponse();

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
      'conversation-123',
      'init boom after res.json',
      1000,
    );
    expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
    expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
    expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
  });

  it('finalizes and disposes a client aborted during initialization before releasing the slot', async () => {
    const abortController = new AbortController();
    let resolveCompletion;
    let signalCompletionStarted;
    const completionStarted = new Promise((resolve) => {
      signalCompletionStarted = resolve;
    });
    mockGenerationJobManager.createJob.mockResolvedValue({
      createdAt: 1000,
      readyPromise: Promise.resolve(),
      abortController,
      emitter: { on: jest.fn() },
    });
    mockGenerationJobManager.completeJob.mockImplementation(() => {
      signalCompletionStarted();
      return new Promise((resolve) => {
        resolveCompletion = resolve;
      });
    });
    const client = { options: {} };
    const initializeClient = jest.fn(async ({ signal }) => {
      expect(signal).toBe(abortController.signal);
      abortController.abort();
      return { client };
    });
    const conversationId = 'conversation-123';
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Stop during initialization.',
        messageId: 'user-msg',
        conversationId,
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = createResumableResponse();

    const controllerPromise = AgentController(req, res, jest.fn(), initializeClient, null);
    await completionStarted;

    expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
      conversationId,
      'Request aborted during initialization',
      1000,
    );
    expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
    expect(mockDisposeClient).not.toHaveBeenCalled();

    resolveCompletion();
    await controllerPromise;

    expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
    expect(mockDisposeClient).toHaveBeenCalledTimes(1);
    expect(mockDisposeClient).toHaveBeenCalledWith(client);
    expect(mockStartupTelemetry.end).toHaveBeenCalledWith('aborted');
  });

  it('awaits background error finalization before releasing the slot and always disposes', async () => {
    const generationError = new Error('generation failed');
    let rejectCompletion;
    let signalCompletionStarted;
    const completionStarted = new Promise((resolve) => {
      signalCompletionStarted = resolve;
    });
    mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
    mockGenerationJobManager.completeJob.mockImplementation(() => {
      signalCompletionStarted();
      return new Promise((_, reject) => {
        rejectCompletion = reject;
      });
    });
    const client = {
      options: {},
      sendMessage: jest.fn().mockRejectedValue(generationError),
    };
    const initializeClient = jest.fn().mockResolvedValue({ client });
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Fail after initialization.',
        messageId: 'user-msg',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = createResumableResponse();

    await AgentController(req, res, jest.fn(), initializeClient, null);
    await completionStarted;

    expect(mockGenerationJobManager.emitError).toHaveBeenCalledWith(
      'conversation-123',
      generationError.message,
      1000,
    );
    expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
    expect(mockDisposeClient).not.toHaveBeenCalled();

    rejectCompletion(new Error('store failed'));
    await nextTick();

    expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
      'conversation-123',
      generationError.message,
      1000,
    );
    expect(mockGenerationJobManager.completeJob.mock.invocationCallOrder[0]).toBeLessThan(
      mockDecrementPendingRequest.mock.invocationCallOrder[0],
    );
    expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
    expect(mockDisposeClient).toHaveBeenCalledWith(client);
  });

  it('proceeds to create the job when it wins the idempotency claim', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Fresh submission.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = {
      headersSent: true,
      json: jest.fn(() => {
        res.headersSent = true;
      }),
      status: jest.fn(() => res),
      set: jest.fn(),
    };

    await AgentController(req, res, jest.fn(), initializeClient, null);

    expect(mockCheckAndIncrementPendingRequest).toHaveBeenCalledWith('user-123');
    expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
      'conversation-123',
      'user-123',
      'conversation-123',
      expect.objectContaining({
        startupTelemetry: mockStartupTelemetry,
        initialMetadata: expect.objectContaining({
          conversationId: 'conversation-123',
          endpoint: 'agents',
        }),
      }),
    );
  });

  it('releases the idempotency claim on a 429 only when it won the claim', async () => {
    mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
    mockCheckAndIncrementPendingRequest.mockResolvedValue({
      allowed: false,
      pendingRequests: 3,
      limit: 2,
    });
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Over the limit.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };

    await AgentController(req, res, jest.fn(), jest.fn(), null);

    expect(res.status).toHaveBeenCalledWith(429);
    expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
    expect(mockStartupTelemetry.end).toHaveBeenCalledWith('rejected');
  });

  it('does not release a claim it never won when a fail-open duplicate hits the limiter', async () => {
    mockGenerationJobManager.claimGeneration.mockRejectedValue(new Error('redis down'));
    mockCheckAndIncrementPendingRequest.mockResolvedValue({
      allowed: false,
      pendingRequests: 3,
      limit: 2,
    });
    const req = {
      user: { id: 'user-123' },
      body: {
        text: 'Duplicate while the original runs.',
        messageId: 'user-msg',
        clientRequestId: 'req-abc',
        conversationId: 'conversation-123',
        endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
      },
      config: {},
    };
    const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };

    await AgentController(req, res, jest.fn(), jest.fn(), null);

    expect(res.status).toHaveBeenCalledWith(429);
    expect(mockGenerationJobManager.releaseGeneration).not.toHaveBeenCalled();
  });
});