mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
73699b5c25
* perf: reduce agent chat startup latency * test: align Redis stream readiness assertions * perf: overlap remaining agent startup work * perf: persist initial agent job metadata atomically * test: add agent startup latency benchmark * fix: harden resumable agent stream lifecycle * fix: isolate replacement stream lifecycles * fix: preserve terminal stream epochs
1189 lines
40 KiB
JavaScript
1189 lines
40 KiB
JavaScript
const { EventEmitter } = require('events');
|
|
|
|
const mockLogger = {
|
|
debug: jest.fn(),
|
|
warn: jest.fn(),
|
|
error: jest.fn(),
|
|
info: jest.fn(),
|
|
};
|
|
|
|
const mockGenerationJobManager = {
|
|
createJob: jest.fn(),
|
|
emitError: jest.fn(),
|
|
completeJob: jest.fn(),
|
|
getResumeState: jest.fn(),
|
|
updateMetadata: jest.fn(),
|
|
claimGeneration: jest.fn(),
|
|
releaseGeneration: jest.fn(),
|
|
hasJob: jest.fn(),
|
|
steering: {
|
|
closeAndDrain: jest.fn(),
|
|
park: jest.fn(),
|
|
},
|
|
};
|
|
|
|
const mockCheckAndIncrementPendingRequest = jest.fn();
|
|
const mockDecrementPendingRequest = jest.fn();
|
|
const mockGetViolationInfo = jest.fn(() => ({
|
|
type: 'concurrent',
|
|
limit: 2,
|
|
pendingRequests: 3,
|
|
score: 1,
|
|
}));
|
|
const mockFilterPersistableAbortContent = jest.fn((content) =>
|
|
content.filter((part) => part?.type !== 'tool_call'),
|
|
);
|
|
const mockGetConvo = jest.fn();
|
|
const mockGetMessages = jest.fn();
|
|
const mockSaveMessage = jest.fn();
|
|
const mockStartupTelemetry = {
|
|
mark: jest.fn(),
|
|
setStreamId: jest.fn(),
|
|
recordGenerationEvent: jest.fn(),
|
|
end: jest.fn(),
|
|
};
|
|
const mockGetAgentStartupTelemetry = jest.fn(() => mockStartupTelemetry);
|
|
const mockAcceptAgentStartupTelemetry = jest.fn();
|
|
let mockMCPContexts = new WeakMap();
|
|
|
|
const mockCreateMCPRequestContext = jest.fn(() => ({
|
|
connections: new Map(),
|
|
pending: new Map(),
|
|
cleanupStarted: false,
|
|
cleanupOnResponse: false,
|
|
responseCleanupAttached: false,
|
|
}));
|
|
const mockGetMCPRequestContext = jest.fn((req) => {
|
|
if (!req) {
|
|
return undefined;
|
|
}
|
|
|
|
let context = mockMCPContexts.get(req);
|
|
if (!context) {
|
|
context = mockCreateMCPRequestContext();
|
|
mockMCPContexts.set(req, context);
|
|
}
|
|
|
|
return context.cleanupStarted ? undefined : context;
|
|
});
|
|
const mockCleanupMCPRequestContext = jest.fn(async (context) => {
|
|
if (!context || context.cleanupStarted) {
|
|
return;
|
|
}
|
|
|
|
context.cleanupStarted = true;
|
|
const connections = new Set(context.connections.values());
|
|
const settled = await Promise.allSettled(context.pending.values());
|
|
for (const result of settled) {
|
|
if (result.status === 'fulfilled' && result.value) {
|
|
connections.add(result.value);
|
|
}
|
|
}
|
|
|
|
await Promise.allSettled(Array.from(connections).map((connection) => connection.disconnect?.()));
|
|
context.connections.clear();
|
|
context.pending.clear();
|
|
});
|
|
const mockCleanupMCPRequestContextForReq = jest.fn(async (req) => {
|
|
const context = mockMCPContexts.get(req);
|
|
if (!context) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await mockCleanupMCPRequestContext(context);
|
|
} finally {
|
|
mockMCPContexts.delete(req);
|
|
}
|
|
});
|
|
|
|
jest.mock('@librechat/data-schemas', () => ({
|
|
logger: mockLogger,
|
|
}));
|
|
|
|
jest.mock('@librechat/api', () => ({
|
|
sendEvent: jest.fn(),
|
|
getViolationInfo: (...args) => mockGetViolationInfo(...args),
|
|
buildMessageFiles: jest.fn(() => []),
|
|
resolveTitleTiming: jest.fn(() => 'immediate'),
|
|
resolveConversationAnchor: jest.requireActual('@librechat/api').resolveConversationAnchor,
|
|
GenerationJobManager: mockGenerationJobManager,
|
|
getReferencedQuotes: jest.fn((quotes) => {
|
|
if (!Array.isArray(quotes)) {
|
|
return null;
|
|
}
|
|
const normalized = quotes
|
|
.filter((quote) => typeof quote === 'string' && quote.trim().length > 0)
|
|
.map((quote) => quote.trim());
|
|
return normalized.length > 0 ? normalized : null;
|
|
}),
|
|
cleanupMCPRequestContext: (...args) => mockCleanupMCPRequestContext(...args),
|
|
createMCPRequestContext: (...args) => mockCreateMCPRequestContext(...args),
|
|
getMCPRequestContext: (...args) => mockGetMCPRequestContext(...args),
|
|
filterPersistableAbortContent: (...args) => mockFilterPersistableAbortContent(...args),
|
|
cleanupMCPRequestContextForReq: (...args) => mockCleanupMCPRequestContextForReq(...args),
|
|
decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
|
|
sanitizeMessageForTransmit: jest.fn((message) => message),
|
|
checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
|
|
getAgentStartupTelemetry: (...args) => mockGetAgentStartupTelemetry(...args),
|
|
acceptAgentStartupTelemetry: (...args) => mockAcceptAgentStartupTelemetry(...args),
|
|
isUnpersistedPreliminaryParent: async ({
|
|
userId,
|
|
conversationId,
|
|
parentMessageId,
|
|
getMessages,
|
|
}) => {
|
|
if (typeof parentMessageId !== 'string' || !parentMessageId.endsWith('_')) {
|
|
return false;
|
|
}
|
|
|
|
const filter = { user: userId, messageId: parentMessageId };
|
|
if (conversationId && conversationId !== 'new') {
|
|
filter.conversationId = conversationId;
|
|
}
|
|
|
|
const messages = await getMessages(filter, '_id');
|
|
return messages.length === 0;
|
|
},
|
|
}));
|
|
|
|
jest.mock('~/server/cleanup', () => ({
|
|
disposeClient: jest.fn(),
|
|
clientRegistry: null,
|
|
requestDataMap: {
|
|
set: jest.fn(),
|
|
},
|
|
}));
|
|
|
|
jest.mock('~/server/middleware', () => ({
|
|
handleAbortError: jest.fn(() => Promise.resolve()),
|
|
}));
|
|
|
|
jest.mock('~/cache', () => ({
|
|
logViolation: jest.fn(),
|
|
}));
|
|
|
|
jest.mock('~/models', () => ({
|
|
saveMessage: (...args) => mockSaveMessage(...args),
|
|
getMessages: (...args) => mockGetMessages(...args),
|
|
getConvo: (...args) => mockGetConvo(...args),
|
|
}));
|
|
|
|
const AgentController = require('../request');
|
|
const { disposeClient: mockDisposeClient } = require('~/server/cleanup');
|
|
const { getMCPRequestContext } = require('~/server/services/MCPRequestContext');
|
|
|
|
function createResumableResponse() {
|
|
const res = new EventEmitter();
|
|
res.headersSent = false;
|
|
res.writableEnded = false;
|
|
res.finished = false;
|
|
res.destroyed = false;
|
|
res.json = jest.fn(() => {
|
|
res.headersSent = true;
|
|
res.writableEnded = true;
|
|
res.finished = true;
|
|
res.emit('finish');
|
|
return res;
|
|
});
|
|
res.status = jest.fn(() => res);
|
|
return res;
|
|
}
|
|
|
|
function nextTick() {
|
|
return new Promise((resolve) => setImmediate(resolve));
|
|
}
|
|
|
|
describe('ResumableAgentController resume metadata', () => {
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
mockMCPContexts = new WeakMap();
|
|
mockCheckAndIncrementPendingRequest.mockResolvedValue({ allowed: true });
|
|
mockDecrementPendingRequest.mockResolvedValue(undefined);
|
|
mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' });
|
|
mockGetMessages.mockResolvedValue([]);
|
|
mockGenerationJobManager.createJob.mockResolvedValue({
|
|
createdAt: 1000,
|
|
readyPromise: Promise.resolve(),
|
|
abortController: new AbortController(),
|
|
emitter: { on: jest.fn() },
|
|
});
|
|
mockGenerationJobManager.getResumeState.mockResolvedValue(null);
|
|
mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
|
|
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
|
|
mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);
|
|
mockGenerationJobManager.hasJob.mockResolvedValue(true);
|
|
mockGenerationJobManager.steering.closeAndDrain.mockResolvedValue([]);
|
|
mockGenerationJobManager.steering.park.mockResolvedValue(undefined);
|
|
mockSaveMessage.mockResolvedValue({});
|
|
});
|
|
|
|
it('rejects an underscore-suffixed parent that is not persisted', async () => {
|
|
const conversationId = 'conversation-123';
|
|
const initializeClient = jest.fn();
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Follow up too early.',
|
|
messageId: 'follow-up-user',
|
|
parentMessageId: 'pending-response_',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
modelOptions: { model: 'gpt-3.5-turbo' },
|
|
},
|
|
},
|
|
config: {},
|
|
};
|
|
const res = {
|
|
json: jest.fn(),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGetMessages).toHaveBeenCalledWith(
|
|
{ user: 'user-123', messageId: 'pending-response_', conversationId },
|
|
'_id',
|
|
);
|
|
expect(res.status).toHaveBeenCalledWith(409);
|
|
expect(res.json).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
error: expect.stringContaining('selected parent response is still being saved'),
|
|
}),
|
|
);
|
|
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
|
|
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
|
|
expect(initializeClient).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('allows an underscore-suffixed parent when it is already persisted', async () => {
|
|
const conversationId = 'conversation-123';
|
|
mockGetMessages.mockResolvedValue([{ _id: 'persisted-parent' }]);
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Follow up to persisted underscore id.',
|
|
messageId: 'follow-up-user',
|
|
parentMessageId: 'persisted-response_',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
modelOptions: { model: 'gpt-3.5-turbo' },
|
|
},
|
|
},
|
|
config: {},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGetMessages).toHaveBeenCalledWith(
|
|
{ user: 'user-123', messageId: 'persisted-response_', conversationId },
|
|
'_id',
|
|
);
|
|
expect(res.status).not.toHaveBeenCalledWith(409);
|
|
expect(mockCheckAndIncrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
|
|
conversationId,
|
|
'user-123',
|
|
conversationId,
|
|
expect.objectContaining({
|
|
startupTelemetry: mockStartupTelemetry,
|
|
initialMetadata: expect.objectContaining({
|
|
conversationId,
|
|
endpoint: 'agents',
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('creates the job with the in-flight turn before MCP initialization can emit OAuth', async () => {
|
|
const conversationId = 'conversation-123';
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Check Google Workspace availability.',
|
|
messageId: 'follow-up-user',
|
|
parentMessageId: 'original-response',
|
|
conversationId,
|
|
isTemporary: true,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
iconURL: 'https://example.com/spec-icon.png',
|
|
modelOptions: { model: 'gpt-3.5-turbo' },
|
|
},
|
|
},
|
|
config: {},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
|
|
conversationId,
|
|
'user-123',
|
|
conversationId,
|
|
{
|
|
startupTelemetry: mockStartupTelemetry,
|
|
initialMetadata: {
|
|
conversationId,
|
|
endpoint: 'agents',
|
|
iconURL: 'https://example.com/spec-icon.png',
|
|
model: 'gpt-3.5-turbo',
|
|
agent_id: undefined,
|
|
isTemporary: true,
|
|
responseMessageId: 'follow-up-user_',
|
|
userMessage: {
|
|
messageId: 'follow-up-user',
|
|
parentMessageId: 'original-response',
|
|
conversationId,
|
|
text: 'Check Google Workspace availability.',
|
|
},
|
|
},
|
|
},
|
|
);
|
|
expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(
|
|
initializeClient.mock.invocationCallOrder[0],
|
|
);
|
|
expect(mockGenerationJobManager.updateMetadata).not.toHaveBeenCalled();
|
|
const startupMilestones = mockStartupTelemetry.mark.mock.calls.map(([milestone]) => milestone);
|
|
expect(startupMilestones.slice(0, 2)).toEqual(['request_admitted', 'job_created']);
|
|
expect(new Set(startupMilestones.slice(2))).toEqual(
|
|
new Set(['conversation_resolved', 'metadata_persisted']),
|
|
);
|
|
expect(mockAcceptAgentStartupTelemetry).toHaveBeenCalledWith(req, conversationId);
|
|
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
|
|
});
|
|
|
|
it('prefetches conversation state before admission and joins it with job metadata', async () => {
|
|
let resolveConversation;
|
|
let signalMetadataStarted;
|
|
const conversationPromise = new Promise((resolve) => {
|
|
resolveConversation = resolve;
|
|
});
|
|
const metadataStarted = new Promise((resolve) => {
|
|
signalMetadataStarted = resolve;
|
|
});
|
|
mockGetConvo.mockReturnValue(conversationPromise);
|
|
mockGenerationJobManager.createJob.mockImplementation(() => {
|
|
signalMetadataStarted();
|
|
return Promise.resolve({
|
|
createdAt: 1000,
|
|
readyPromise: Promise.resolve(),
|
|
abortController: new AbortController(),
|
|
emitter: { on: jest.fn() },
|
|
});
|
|
});
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop after startup reads'));
|
|
const conversationId = 'conversation-123';
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Run independent startup work together.',
|
|
messageId: 'user-message',
|
|
parentMessageId: 'parent-message',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
modelOptions: { model: 'gpt-4.1' },
|
|
},
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
const controllerPromise = AgentController(req, res, jest.fn(), initializeClient, null);
|
|
expect(mockGetConvo).toHaveBeenCalledWith('user-123', conversationId);
|
|
await metadataStarted;
|
|
await nextTick();
|
|
|
|
expect(mockGetConvo.mock.invocationCallOrder[0]).toBeLessThan(
|
|
mockCheckAndIncrementPendingRequest.mock.invocationCallOrder[0],
|
|
);
|
|
expect(res.json).toHaveBeenCalledWith({
|
|
streamId: conversationId,
|
|
conversationId,
|
|
status: 'started',
|
|
});
|
|
expect(initializeClient).not.toHaveBeenCalled();
|
|
|
|
resolveConversation({ createdAt: '2026-06-07T00:00:00.000Z' });
|
|
await controllerPromise;
|
|
|
|
expect(initializeClient).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('keeps request-scoped MCP connections until resumable initialization finishes', async () => {
|
|
const conversationId = 'conversation-123';
|
|
const disconnect = jest.fn().mockResolvedValue(undefined);
|
|
const initializeClient = jest.fn(async ({ req, res }) => {
|
|
const context = getMCPRequestContext(req, res);
|
|
context.connections.set('mcp-server', { disconnect });
|
|
|
|
await nextTick();
|
|
expect(disconnect).not.toHaveBeenCalled();
|
|
|
|
throw new Error('stop after request-scoped MCP connection');
|
|
});
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Use a BODY-scoped MCP server.',
|
|
messageId: 'user-message',
|
|
parentMessageId: 'parent-message',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
modelOptions: { model: 'gpt-4.1' },
|
|
},
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(res.json).toHaveBeenCalledWith({
|
|
streamId: conversationId,
|
|
conversationId,
|
|
status: 'started',
|
|
});
|
|
expect(disconnect).toHaveBeenCalledTimes(1);
|
|
expect(disconnect.mock.invocationCallOrder[0]).toBeLessThan(
|
|
mockDecrementPendingRequest.mock.invocationCallOrder[0],
|
|
);
|
|
});
|
|
|
|
it('stores model spec icon fallbacks and agent ids in early resume metadata', async () => {
|
|
const conversationId = 'conversation-123';
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Use the resume spec.',
|
|
messageId: 'follow-up-user',
|
|
parentMessageId: 'original-response',
|
|
conversationId,
|
|
isTemporary: true,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
spec: 'agent-spec',
|
|
agent_id: 'agent_resume_spec',
|
|
model_parameters: { model: 'gpt-4.1' },
|
|
},
|
|
},
|
|
config: {
|
|
modelSpecs: {
|
|
list: [
|
|
{
|
|
name: 'agent-spec',
|
|
preset: {
|
|
endpoint: 'openAI',
|
|
iconURL: 'https://example.com/preset-icon.png',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
|
|
conversationId,
|
|
'user-123',
|
|
conversationId,
|
|
expect.objectContaining({
|
|
initialMetadata: expect.objectContaining({
|
|
iconURL: 'https://example.com/preset-icon.png',
|
|
model: 'agent_resume_spec',
|
|
agent_id: 'agent_resume_spec',
|
|
isTemporary: true,
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('falls back to the model spec preset endpoint when no icon URL is configured', async () => {
|
|
const conversationId = 'conversation-123';
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Use the endpoint icon.',
|
|
messageId: 'follow-up-user',
|
|
parentMessageId: 'original-response',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
spec: 'endpoint-icon-spec',
|
|
model_parameters: { model: 'gpt-4.1' },
|
|
},
|
|
},
|
|
config: {
|
|
modelSpecs: {
|
|
list: [
|
|
{
|
|
name: 'endpoint-icon-spec',
|
|
preset: {
|
|
endpoint: 'anthropic',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
|
|
conversationId,
|
|
'user-123',
|
|
conversationId,
|
|
expect.objectContaining({
|
|
initialMetadata: expect.objectContaining({
|
|
iconURL: 'anthropic',
|
|
model: 'gpt-4.1',
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('filters OAuth prompts before saving partial responses on disconnect', async () => {
|
|
const conversationId = 'conversation-123';
|
|
let allSubscribersLeftHandler;
|
|
mockGenerationJobManager.createJob.mockResolvedValue({
|
|
createdAt: 1000,
|
|
readyPromise: Promise.resolve(),
|
|
abortController: new AbortController(),
|
|
emitter: {
|
|
on: jest.fn((event, handler) => {
|
|
if (event === 'allSubscribersLeft') {
|
|
allSubscribersLeftHandler = handler;
|
|
}
|
|
}),
|
|
},
|
|
});
|
|
mockGenerationJobManager.getResumeState.mockResolvedValue({
|
|
conversationId,
|
|
responseMessageId: 'response-message',
|
|
iconURL: 'https://example.com/spec-icon.png',
|
|
model: 'gpt-4.1',
|
|
userMessage: {
|
|
messageId: 'user-message',
|
|
parentMessageId: 'parent-message',
|
|
conversationId,
|
|
text: 'Use Google Workspace',
|
|
},
|
|
});
|
|
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop after setup'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Use Google Workspace',
|
|
messageId: 'user-message',
|
|
parentMessageId: 'parent-message',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
iconURL: 'https://example.com/fallback-icon.png',
|
|
modelOptions: { model: 'gpt-3.5-turbo' },
|
|
},
|
|
},
|
|
config: {},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
expect(allSubscribersLeftHandler).toEqual(expect.any(Function));
|
|
|
|
const oauthPart = {
|
|
type: 'tool_call',
|
|
tool_call: {
|
|
name: 'oauth_mcp_Google-Workspace',
|
|
auth: 'https://auth.example.com/oauth',
|
|
},
|
|
};
|
|
const textPart = { type: 'text', text: 'Partial response...' };
|
|
|
|
await allSubscribersLeftHandler([oauthPart, textPart]);
|
|
|
|
expect(mockFilterPersistableAbortContent).toHaveBeenCalledWith([oauthPart, textPart]);
|
|
expect(mockSaveMessage).toHaveBeenCalledWith(
|
|
expect.objectContaining({ userId: 'user-123' }),
|
|
expect.objectContaining({
|
|
content: [textPart],
|
|
iconURL: 'https://example.com/spec-icon.png',
|
|
model: 'gpt-4.1',
|
|
messageId: 'response-message',
|
|
parentMessageId: 'user-message',
|
|
}),
|
|
expect.any(Object),
|
|
);
|
|
});
|
|
|
|
it('uses model spec and agent fallbacks when saving partial responses on disconnect', async () => {
|
|
const conversationId = 'conversation-123';
|
|
let allSubscribersLeftHandler;
|
|
mockGenerationJobManager.createJob.mockResolvedValue({
|
|
createdAt: 1000,
|
|
readyPromise: Promise.resolve(),
|
|
abortController: new AbortController(),
|
|
emitter: {
|
|
on: jest.fn((event, handler) => {
|
|
if (event === 'allSubscribersLeft') {
|
|
allSubscribersLeftHandler = handler;
|
|
}
|
|
}),
|
|
},
|
|
});
|
|
mockGenerationJobManager.getResumeState.mockResolvedValue({
|
|
conversationId,
|
|
responseMessageId: 'response-message',
|
|
userMessage: {
|
|
messageId: 'user-message',
|
|
parentMessageId: 'parent-message',
|
|
conversationId,
|
|
text: 'Use fallback metadata',
|
|
},
|
|
});
|
|
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop after setup'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Use fallback metadata',
|
|
messageId: 'user-message',
|
|
parentMessageId: 'parent-message',
|
|
conversationId,
|
|
endpointOption: {
|
|
endpoint: 'agents',
|
|
spec: 'agent-spec',
|
|
agent_id: 'agent_resume_spec',
|
|
model_parameters: { model: 'gpt-4.1' },
|
|
},
|
|
},
|
|
config: {
|
|
modelSpecs: {
|
|
list: [
|
|
{
|
|
name: 'agent-spec',
|
|
preset: {
|
|
endpoint: 'openAI',
|
|
iconURL: 'https://example.com/preset-icon.png',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
expect(allSubscribersLeftHandler).toEqual(expect.any(Function));
|
|
|
|
const textPart = { type: 'text', text: 'Partial response...' };
|
|
await allSubscribersLeftHandler([textPart]);
|
|
|
|
expect(mockSaveMessage).toHaveBeenCalledWith(
|
|
expect.objectContaining({ userId: 'user-123' }),
|
|
expect.objectContaining({
|
|
content: [textPart],
|
|
iconURL: 'https://example.com/preset-icon.png',
|
|
model: 'agent_resume_spec',
|
|
messageId: 'response-message',
|
|
parentMessageId: 'user-message',
|
|
}),
|
|
expect.any(Object),
|
|
);
|
|
});
|
|
|
|
it('dedups a retried start-generation request to the original stream', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({
|
|
claimed: false,
|
|
existing: { streamId: 'orig-stream', conversationId: 'orig-convo' },
|
|
});
|
|
mockGenerationJobManager.hasJob.mockResolvedValue(true);
|
|
const initializeClient = jest.fn();
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Retried after a lost response.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(res.json).toHaveBeenCalledWith({
|
|
streamId: 'orig-stream',
|
|
conversationId: 'orig-convo',
|
|
status: 'resumed',
|
|
});
|
|
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
|
|
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
|
|
expect(initializeClient).not.toHaveBeenCalled();
|
|
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('deduplicated');
|
|
});
|
|
|
|
it('resumes when the job is missing but the claim is old (original completed and was cleaned up)', async () => {
|
|
// An old claim with no job means the original already ran and was cleaned up; the deduped
|
|
// response must attach (client 404 handler refetches) rather than loop on readiness.
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({
|
|
claimed: false,
|
|
existing: {
|
|
streamId: 'orig-stream',
|
|
conversationId: 'orig-convo',
|
|
claimedAt: Date.now() - 60000,
|
|
},
|
|
});
|
|
mockGenerationJobManager.hasJob.mockResolvedValue(false);
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Retry after a fast, already-cleaned-up generation.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
|
|
|
|
await AgentController(req, res, jest.fn(), jest.fn(), null);
|
|
|
|
expect(res.json).toHaveBeenCalledWith({
|
|
streamId: 'orig-stream',
|
|
conversationId: 'orig-convo',
|
|
status: 'resumed',
|
|
});
|
|
expect(res.status).not.toHaveBeenCalledWith(503);
|
|
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('returns 503 SERVER_NOT_READY when a fresh claim still has no job (winner is between claim and createJob)', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({
|
|
claimed: false,
|
|
existing: {
|
|
streamId: 'orig-stream',
|
|
conversationId: 'orig-convo',
|
|
claimedAt: Date.now(),
|
|
},
|
|
});
|
|
mockGenerationJobManager.hasJob.mockResolvedValue(false);
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Concurrent duplicate before the winner wrote its job.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
|
|
|
|
await AgentController(req, res, jest.fn(), jest.fn(), null);
|
|
|
|
expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
|
|
expect(res.status).toHaveBeenCalledWith(503);
|
|
expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
|
|
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('never starts a second generation when the job lookup fails for a confirmed duplicate', async () => {
|
|
// A store hiccup while checking an existing claim must not fail open into createJob.
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({
|
|
claimed: false,
|
|
existing: { streamId: 'orig-stream', conversationId: 'orig-convo', claimedAt: Date.now() },
|
|
});
|
|
mockGenerationJobManager.hasJob.mockRejectedValue(new Error('redis down'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Duplicate during a Redis hiccup.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
|
|
|
|
await AgentController(req, res, jest.fn(), jest.fn(), null);
|
|
|
|
expect(res.status).toHaveBeenCalledWith(503);
|
|
expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
|
|
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('does not finalize an unscoped generation when job creation rejects before returning', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
mockGenerationJobManager.createJob.mockRejectedValue(new Error('create failed before return'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Fail before receiving a job epoch.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
await AgentController(req, res, jest.fn(), jest.fn(), null);
|
|
|
|
expect(res.status).toHaveBeenCalledWith(500);
|
|
expect(res.json).toHaveBeenCalledWith({ error: 'create failed before return' });
|
|
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
|
|
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
|
|
expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
|
|
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
});
|
|
|
|
it('finalizes the failed job before releasing the idempotency claim', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Start fails after the initial JSON.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
|
|
'conversation-123',
|
|
expect.any(String),
|
|
1000,
|
|
);
|
|
expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
|
|
// completeJob must finalize the failed job BEFORE the claim is released, or a racing
|
|
// retry could win the key, createJob the same streamId, and be aborted by this completeJob.
|
|
expect(mockGenerationJobManager.completeJob.mock.invocationCallOrder[0]).toBeLessThan(
|
|
mockGenerationJobManager.releaseGeneration.mock.invocationCallOrder[0],
|
|
);
|
|
});
|
|
|
|
it('still releases the claim and pending slot when completeJob fails during init-error cleanup', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
mockGenerationJobManager.completeJob.mockRejectedValue(new Error('store hiccup'));
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Start fails while the store is degraded.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
// A completeJob rejection must not wedge the retry behind the claim or leak the slot.
|
|
expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
|
|
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
});
|
|
|
|
it('still finalizes and releases when streaming the initialization error fails', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Start fails while Redis publish is degraded.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
|
|
'conversation-123',
|
|
'init boom after res.json',
|
|
1000,
|
|
);
|
|
expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
|
|
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
|
|
});
|
|
|
|
it('finalizes and disposes a client aborted during initialization before releasing the slot', async () => {
|
|
const abortController = new AbortController();
|
|
let resolveCompletion;
|
|
let signalCompletionStarted;
|
|
const completionStarted = new Promise((resolve) => {
|
|
signalCompletionStarted = resolve;
|
|
});
|
|
mockGenerationJobManager.createJob.mockResolvedValue({
|
|
createdAt: 1000,
|
|
readyPromise: Promise.resolve(),
|
|
abortController,
|
|
emitter: { on: jest.fn() },
|
|
});
|
|
mockGenerationJobManager.completeJob.mockImplementation(() => {
|
|
signalCompletionStarted();
|
|
return new Promise((resolve) => {
|
|
resolveCompletion = resolve;
|
|
});
|
|
});
|
|
const client = { options: {} };
|
|
const initializeClient = jest.fn(async ({ signal }) => {
|
|
expect(signal).toBe(abortController.signal);
|
|
abortController.abort();
|
|
return { client };
|
|
});
|
|
const conversationId = 'conversation-123';
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Stop during initialization.',
|
|
messageId: 'user-msg',
|
|
conversationId,
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
const controllerPromise = AgentController(req, res, jest.fn(), initializeClient, null);
|
|
await completionStarted;
|
|
|
|
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
|
|
conversationId,
|
|
'Request aborted during initialization',
|
|
1000,
|
|
);
|
|
expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
|
|
expect(mockDisposeClient).not.toHaveBeenCalled();
|
|
|
|
resolveCompletion();
|
|
await controllerPromise;
|
|
|
|
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
|
|
expect(mockDisposeClient).toHaveBeenCalledWith(client);
|
|
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('aborted');
|
|
});
|
|
|
|
it('awaits background error finalization before releasing the slot and always disposes', async () => {
|
|
const generationError = new Error('generation failed');
|
|
let rejectCompletion;
|
|
let signalCompletionStarted;
|
|
const completionStarted = new Promise((resolve) => {
|
|
signalCompletionStarted = resolve;
|
|
});
|
|
mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
|
|
mockGenerationJobManager.completeJob.mockImplementation(() => {
|
|
signalCompletionStarted();
|
|
return new Promise((_, reject) => {
|
|
rejectCompletion = reject;
|
|
});
|
|
});
|
|
const client = {
|
|
options: {},
|
|
sendMessage: jest.fn().mockRejectedValue(generationError),
|
|
};
|
|
const initializeClient = jest.fn().mockResolvedValue({ client });
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Fail after initialization.',
|
|
messageId: 'user-msg',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = createResumableResponse();
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
await completionStarted;
|
|
|
|
expect(mockGenerationJobManager.emitError).toHaveBeenCalledWith(
|
|
'conversation-123',
|
|
generationError.message,
|
|
1000,
|
|
);
|
|
expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
|
|
expect(mockDisposeClient).not.toHaveBeenCalled();
|
|
|
|
rejectCompletion(new Error('store failed'));
|
|
await nextTick();
|
|
|
|
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
|
|
'conversation-123',
|
|
generationError.message,
|
|
1000,
|
|
);
|
|
expect(mockGenerationJobManager.completeJob.mock.invocationCallOrder[0]).toBeLessThan(
|
|
mockDecrementPendingRequest.mock.invocationCallOrder[0],
|
|
);
|
|
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
expect(mockDisposeClient).toHaveBeenCalledWith(client);
|
|
});
|
|
|
|
it('proceeds to create the job when it wins the idempotency claim', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Fresh submission.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = {
|
|
headersSent: true,
|
|
json: jest.fn(() => {
|
|
res.headersSent = true;
|
|
}),
|
|
status: jest.fn(() => res),
|
|
set: jest.fn(),
|
|
};
|
|
|
|
await AgentController(req, res, jest.fn(), initializeClient, null);
|
|
|
|
expect(mockCheckAndIncrementPendingRequest).toHaveBeenCalledWith('user-123');
|
|
expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
|
|
'conversation-123',
|
|
'user-123',
|
|
'conversation-123',
|
|
expect.objectContaining({
|
|
startupTelemetry: mockStartupTelemetry,
|
|
initialMetadata: expect.objectContaining({
|
|
conversationId: 'conversation-123',
|
|
endpoint: 'agents',
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('releases the idempotency claim on a 429 only when it won the claim', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
|
|
mockCheckAndIncrementPendingRequest.mockResolvedValue({
|
|
allowed: false,
|
|
pendingRequests: 3,
|
|
limit: 2,
|
|
});
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Over the limit.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
|
|
|
|
await AgentController(req, res, jest.fn(), jest.fn(), null);
|
|
|
|
expect(res.status).toHaveBeenCalledWith(429);
|
|
expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
|
|
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('rejected');
|
|
});
|
|
|
|
it('does not release a claim it never won when a fail-open duplicate hits the limiter', async () => {
|
|
mockGenerationJobManager.claimGeneration.mockRejectedValue(new Error('redis down'));
|
|
mockCheckAndIncrementPendingRequest.mockResolvedValue({
|
|
allowed: false,
|
|
pendingRequests: 3,
|
|
limit: 2,
|
|
});
|
|
const req = {
|
|
user: { id: 'user-123' },
|
|
body: {
|
|
text: 'Duplicate while the original runs.',
|
|
messageId: 'user-msg',
|
|
clientRequestId: 'req-abc',
|
|
conversationId: 'conversation-123',
|
|
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
|
|
},
|
|
config: {},
|
|
};
|
|
const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
|
|
|
|
await AgentController(req, res, jest.fn(), jest.fn(), null);
|
|
|
|
expect(res.status).toHaveBeenCalledWith(429);
|
|
expect(mockGenerationJobManager.releaseGeneration).not.toHaveBeenCalled();
|
|
});
|
|
});
|