mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-09-06 22:49:29 +00:00
🛎️ fix: Enroll Remote Agent Runs in the Generation Lifecycle (#15349)
* fix: enroll remote agent runs in generation lifecycle * fix: close remote lifecycle ownership gaps * fix: close remote conversation drain races * fix: fence remote runs during conversation deletion * fix: reconcile remote deletion and settlement races * fix: complete owner deletion recovery * fix: preserve remote cleanup receipts * fix: consume deletion receipts before cleanup * fix: expose idempotent deletion option * chore: sort remote lifecycle imports
This commit is contained in:
Danny Avila
GitHub
parent
8fcab7e44f
commit
70f735336d
22 changed files with 1550 additions and 184 deletions
|
|
@ -1,6 +1,7 @@
|
|||
# Domain language
|
||||
|
||||
- **Agent run envelope**: the versioned, JSON-safe request contract created after ingress authentication and protocol validation but before agent, provider, tool, or MCP initialization. It carries only the validated protocol payload and the minimum trusted principal identifiers. The execution host rehydrates all runtime state from those identifiers.
|
||||
- **Agent execution enrollment**: the durable, protocol-neutral lifecycle authority for an admitted Agent run. It is created under the authenticated user and tenant before user-owned initialization, rechecks the shared owner-deletion admission fence after registration, exposes the only provider abort signal, fences exact provider start, terminalizes the run, waits for every trailing usage, artifact, and stored-response write, and acknowledges provider drain last. A transient terminalization failure is reconciled after trailing writes; provider drain is never acknowledged while the exact job remains nonterminal. Delete-all holds the owner fence, drains every owner run before selecting its first persistence snapshot, and repeats both the drain and an idempotent owner-persistence sweep after any recovered fence lapse before releasing admission. Exact-conversation deletion additionally performs an unconditional idempotent cleanup over its immutable deleted-ID set because a fully drained run may leave the active index after racing the first delete; only the explicit empty result is benign, while storage failures remain fatal. Chat Completions, Responses, Channels, and future ingress adapters share this authority without moving LibreChat persistence policy into the Agents SDK.
|
||||
- **Agent turn execution plan**: the immutable, request-local decision compiled once after authentication, agent resolution, and tool initialization. It records the trusted turn origin, conversation lineage, pause capability, binding/action context, and the preferred checkpoint, history, or fresh state-loading strategy without executing the model or owning persistence. Checkpoint failure falls back to durable history within the same Agents lifecycle.
|
||||
- **Effective agent selection**: the resolved endpoint and agent identity after an enforced model spec is applied. Authorization and agent loading must consume this same identity before the Agent run envelope is initialized.
|
||||
- **MCP runtime request body**: trusted chat identifiers supplied only while an MCP server handles an agent request. It enables request-scoped header placeholders without retaining user-specific request data on a shared server definition.
|
||||
|
|
|
|||
|
|
@ -26,6 +26,27 @@ const mockCompletionUsage = {
|
|||
subagent: { prompt_tokens: 25, completion_tokens: 10, total_tokens: 35 },
|
||||
};
|
||||
const mockBuildCompletionUsage = jest.fn().mockReturnValue(mockCompletionUsage);
|
||||
const mockEnrollAgentExecution = jest.fn();
|
||||
let mockExecution;
|
||||
|
||||
function resetMockExecution() {
|
||||
const controller = new AbortController();
|
||||
mockExecution = {
|
||||
signal: controller.signal,
|
||||
abort: jest.fn((reason) => controller.abort(reason)),
|
||||
track: jest.fn((promise) => promise),
|
||||
beginProviderExecution: jest.fn(async () => {
|
||||
if (controller.signal.aborted) {
|
||||
throw Object.assign(new Error('request disconnected'), {
|
||||
code: 'RUN_REPLACED',
|
||||
status: 409,
|
||||
});
|
||||
}
|
||||
}),
|
||||
settle: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
mockEnrollAgentExecution.mockResolvedValue(mockExecution);
|
||||
}
|
||||
const mockInitialSessions = new Map([['execute_code', { session_id: 'seeded' }]]);
|
||||
const mockGetSafeErrorMetadata = jest.fn((error) => {
|
||||
const status = error?.status ?? error?.statusCode ?? error?.response?.status;
|
||||
|
|
@ -274,6 +295,12 @@ jest.mock('@librechat/api', () => ({
|
|||
userMCPAuthMap: undefined,
|
||||
}),
|
||||
resolveSubagentGraphs: jest.fn().mockResolvedValue(undefined),
|
||||
enrollAgentExecution: (...args) => mockEnrollAgentExecution(...args),
|
||||
waitForAgentExecutionWrites: async (writes) => {
|
||||
const results = await Promise.allSettled(writes);
|
||||
const failure = results.find((result) => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') throw failure.reason;
|
||||
},
|
||||
}));
|
||||
|
||||
jest.mock('~/server/controllers/ModelController', () => ({
|
||||
|
|
@ -365,6 +392,7 @@ jest.mock('~/models', () => ({
|
|||
getConvoFiles: jest.fn().mockResolvedValue([]),
|
||||
getFormattedMemories: jest.fn().mockResolvedValue({ withKeys: '', withoutKeys: '' }),
|
||||
getConvo: jest.fn().mockResolvedValue(null),
|
||||
isSubagentOwnerAdmissible: jest.fn().mockResolvedValue(true),
|
||||
}));
|
||||
|
||||
describe('OpenAIChatCompletionController', () => {
|
||||
|
|
@ -373,6 +401,7 @@ describe('OpenAIChatCompletionController', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
resetMockExecution();
|
||||
|
||||
const controller = require('../openai');
|
||||
OpenAIChatCompletionController = controller.OpenAIChatCompletionController;
|
||||
|
|
@ -389,7 +418,8 @@ describe('OpenAIChatCompletionController', () => {
|
|||
agents: { allowedProviders: ['openAI'] },
|
||||
},
|
||||
},
|
||||
on: jest.fn(),
|
||||
once: jest.fn(),
|
||||
off: jest.fn(),
|
||||
};
|
||||
|
||||
res = {
|
||||
|
|
@ -399,9 +429,85 @@ describe('OpenAIChatCompletionController', () => {
|
|||
flushHeaders: jest.fn(),
|
||||
end: jest.fn(),
|
||||
write: jest.fn(),
|
||||
once: jest.fn(),
|
||||
off: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
it('enrolls, starts, and settles the remote execution lifecycle', async () => {
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
expect(mockEnrollAgentExecution).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: 'chatcmpl-mock-nanoid-123',
|
||||
userId: 'user-123',
|
||||
agentId: 'agent-123',
|
||||
protocol: 'chat.completions',
|
||||
}),
|
||||
);
|
||||
expect(mockExecution.beginProviderExecution).toHaveBeenCalledTimes(1);
|
||||
expect(mockExecution.beginProviderExecution.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
require('@librechat/api').initializeAgent.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockExecution.beginProviderExecution.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockProcessStream.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockExecution.settle).toHaveBeenCalledWith(undefined);
|
||||
expect(res.once).toHaveBeenCalledWith('close', expect.any(Function));
|
||||
expect(res.off).toHaveBeenCalledWith('close', expect.any(Function));
|
||||
});
|
||||
|
||||
it('covers artifact writes when provider execution fails', async () => {
|
||||
const providerError = new Error('provider aborted');
|
||||
const artifactWrite = Promise.resolve(null);
|
||||
const { createToolEndCallback } = require('~/server/controllers/agents/callbacks');
|
||||
createToolEndCallback.mockImplementationOnce(({ artifactPromises }) => {
|
||||
artifactPromises.push(artifactWrite);
|
||||
return jest.fn();
|
||||
});
|
||||
mockProcessStream.mockRejectedValueOnce(providerError);
|
||||
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
expect(mockExecution.track).toHaveBeenCalledWith(expect.any(Promise));
|
||||
expect(mockExecution.track.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockExecution.settle.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockExecution.settle).toHaveBeenCalledWith(providerError);
|
||||
});
|
||||
|
||||
it('does not initialize a provider after disconnecting during enrollment', async () => {
|
||||
let finishEnrollment;
|
||||
mockEnrollAgentExecution.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finishEnrollment = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const request = OpenAIChatCompletionController(req, res);
|
||||
await Promise.resolve();
|
||||
res.once.mock.calls[0][1]();
|
||||
finishEnrollment(mockExecution);
|
||||
await request;
|
||||
|
||||
expect(mockExecution.abort).toHaveBeenCalledTimes(1);
|
||||
expect(mockExecution.beginProviderExecution).toHaveBeenCalledTimes(1);
|
||||
expect(require('@librechat/api').initializeAgent).not.toHaveBeenCalled();
|
||||
expect(mockExecution.settle).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ code: 'RUN_REPLACED' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not treat a consumed request stream as a response disconnect', async () => {
|
||||
req.destroyed = true;
|
||||
|
||||
await OpenAIChatCompletionController(req, res);
|
||||
|
||||
expect(mockExecution.abort).not.toHaveBeenCalled();
|
||||
expect(mockExecution.beginProviderExecution).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resolves saved graph subagents for remote chat-completion runs', async () => {
|
||||
const {
|
||||
initializeAgent,
|
||||
|
|
|
|||
|
|
@ -148,6 +148,27 @@ const mockResponsesUsage = {
|
|||
subagent: { input_tokens: 25, output_tokens: 10, total_tokens: 35 },
|
||||
};
|
||||
const mockBuildResponsesUsage = jest.fn().mockReturnValue(mockResponsesUsage);
|
||||
const mockEnrollAgentExecution = jest.fn();
|
||||
let mockExecution;
|
||||
|
||||
function resetMockExecution() {
|
||||
const controller = new AbortController();
|
||||
mockExecution = {
|
||||
signal: controller.signal,
|
||||
abort: jest.fn((reason) => controller.abort(reason)),
|
||||
track: jest.fn((promise) => promise),
|
||||
beginProviderExecution: jest.fn(async () => {
|
||||
if (controller.signal.aborted) {
|
||||
throw Object.assign(new Error('request disconnected'), {
|
||||
code: 'RUN_REPLACED',
|
||||
status: 409,
|
||||
});
|
||||
}
|
||||
}),
|
||||
settle: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
mockEnrollAgentExecution.mockResolvedValue(mockExecution);
|
||||
}
|
||||
|
||||
jest.mock('nanoid', () => ({
|
||||
nanoid: jest.fn(() => 'mock-nanoid-123'),
|
||||
|
|
@ -340,6 +361,12 @@ jest.mock('@librechat/api', () => ({
|
|||
on_run_step_delta: { handle: jest.fn() },
|
||||
on_chat_model_end: { handle: jest.fn() },
|
||||
}),
|
||||
enrollAgentExecution: (...args) => mockEnrollAgentExecution(...args),
|
||||
waitForAgentExecutionWrites: async (writes) => {
|
||||
const results = await Promise.allSettled(writes);
|
||||
const failure = results.find((result) => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') throw failure.reason;
|
||||
},
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/ToolService', () => ({
|
||||
|
|
@ -441,6 +468,7 @@ jest.mock('~/models', () => ({
|
|||
getFormattedMemories: jest.fn().mockResolvedValue({ withKeys: '', withoutKeys: '' }),
|
||||
saveConvo: jest.fn().mockResolvedValue({}),
|
||||
getConvo: jest.fn().mockResolvedValue(null),
|
||||
isSubagentOwnerAdmissible: jest.fn().mockResolvedValue(true),
|
||||
}));
|
||||
|
||||
let mockGlobalDiscoveredAgentConfigs = null;
|
||||
|
|
@ -451,6 +479,7 @@ describe('createResponse controller', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
resetMockExecution();
|
||||
mockGlobalDiscoveredAgentConfigs = null;
|
||||
require('@librechat/api').inspectContent.mockReset().mockReturnValue(null);
|
||||
|
||||
|
|
@ -469,7 +498,8 @@ describe('createResponse controller', () => {
|
|||
agents: { allowedProviders: ['anthropic'] },
|
||||
},
|
||||
},
|
||||
on: jest.fn(),
|
||||
once: jest.fn(),
|
||||
off: jest.fn(),
|
||||
};
|
||||
|
||||
res = {
|
||||
|
|
@ -479,9 +509,89 @@ describe('createResponse controller', () => {
|
|||
flushHeaders: jest.fn(),
|
||||
end: jest.fn(),
|
||||
write: jest.fn(),
|
||||
once: jest.fn(),
|
||||
off: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
it('enrolls, starts, and settles the remote execution lifecycle', async () => {
|
||||
await createResponse(req, res);
|
||||
|
||||
expect(mockEnrollAgentExecution).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: 'resp_mock-123',
|
||||
userId: 'user-123',
|
||||
agentId: 'agent-123',
|
||||
protocol: 'responses',
|
||||
}),
|
||||
);
|
||||
const { createRun } = require('@librechat/api');
|
||||
const processStream = await createRun.mock.results.at(-1).value;
|
||||
expect(mockExecution.beginProviderExecution).toHaveBeenCalledTimes(1);
|
||||
expect(mockExecution.beginProviderExecution.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
require('@librechat/api').initializeAgent.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockExecution.beginProviderExecution.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
processStream.processStream.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockExecution.settle).toHaveBeenCalledWith(undefined);
|
||||
expect(res.once).toHaveBeenCalledWith('close', expect.any(Function));
|
||||
expect(res.off).toHaveBeenCalledWith('close', expect.any(Function));
|
||||
});
|
||||
|
||||
it('covers artifact writes when provider execution fails', async () => {
|
||||
const providerError = new Error('provider aborted');
|
||||
const artifactWrite = Promise.resolve(null);
|
||||
const processStream = jest.fn().mockRejectedValue(providerError);
|
||||
const { createRun } = require('@librechat/api');
|
||||
const { createToolEndCallback } = require('~/server/controllers/agents/callbacks');
|
||||
createRun.mockResolvedValueOnce({ processStream });
|
||||
createToolEndCallback.mockImplementationOnce(({ artifactPromises }) => {
|
||||
artifactPromises.push(artifactWrite);
|
||||
return jest.fn();
|
||||
});
|
||||
|
||||
await createResponse(req, res);
|
||||
|
||||
expect(mockExecution.track).toHaveBeenCalledWith(expect.any(Promise));
|
||||
expect(mockExecution.track.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockExecution.settle.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(mockExecution.settle).toHaveBeenCalledWith(providerError);
|
||||
});
|
||||
|
||||
it('does not initialize a provider after disconnecting during enrollment', async () => {
|
||||
let finishEnrollment;
|
||||
mockEnrollAgentExecution.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finishEnrollment = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const request = createResponse(req, res);
|
||||
await Promise.resolve();
|
||||
res.once.mock.calls[0][1]();
|
||||
finishEnrollment(mockExecution);
|
||||
await request;
|
||||
|
||||
expect(mockExecution.abort).toHaveBeenCalledTimes(1);
|
||||
expect(mockExecution.beginProviderExecution).toHaveBeenCalledTimes(1);
|
||||
expect(require('@librechat/api').initializeAgent).not.toHaveBeenCalled();
|
||||
expect(mockExecution.settle).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ code: 'RUN_REPLACED' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not treat a consumed request stream as a response disconnect', async () => {
|
||||
req.destroyed = true;
|
||||
|
||||
await createResponse(req, res);
|
||||
|
||||
expect(mockExecution.abort).not.toHaveBeenCalled();
|
||||
expect(mockExecution.beginProviderExecution).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resolves saved graph subagents for remote Responses API runs', async () => {
|
||||
const { initializeAgent, resolveSubagentGraphs } = require('@librechat/api');
|
||||
const primaryConfig = {
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@ const {
|
|||
createOpenAIContentAggregator,
|
||||
isChatCompletionValidationFailure,
|
||||
stripActivityLabelParts,
|
||||
enrollAgentExecution,
|
||||
waitForAgentExecutionWrites,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
buildSummarizationHandlers,
|
||||
|
|
@ -345,18 +347,40 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
`[OpenAI API] Response ${responseId} started for agent ${agentId}, stream: ${request.stream}`,
|
||||
);
|
||||
|
||||
// Set up abort controller
|
||||
const abortController = new AbortController();
|
||||
|
||||
// Handle client disconnect
|
||||
req.on('close', () => {
|
||||
if (!abortController.signal.aborted) {
|
||||
abortController.abort();
|
||||
const conversationId = request.conversation_id ?? nanoid();
|
||||
let execution;
|
||||
let executionError;
|
||||
let responseClosed = res.destroyed === true && res.writableEnded !== true;
|
||||
/** @type {Promise<import('librechat-data-provider').TAttachment | null>[]} */
|
||||
const artifactPromises = [];
|
||||
let artifactWritesCovered = false;
|
||||
const abortOnResponseClose = () => {
|
||||
if (res.writableEnded === true) {
|
||||
return;
|
||||
}
|
||||
responseClosed = true;
|
||||
if (execution && !execution.signal.aborted) {
|
||||
execution.abort();
|
||||
logger.debug('[OpenAI API] Client disconnected, aborting');
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
res.once('close', abortOnResponseClose);
|
||||
try {
|
||||
execution = await enrollAgentExecution({
|
||||
runId: responseId,
|
||||
userId: principal.userId,
|
||||
conversationId,
|
||||
agentId,
|
||||
protocol: 'chat.completions',
|
||||
/** Conversation delete-all uses the shared owner-admission fence. Remote
|
||||
* execution must observe it after durable enrollment and before provider work. */
|
||||
isPrincipalActive: db.isSubagentOwnerAdmissible,
|
||||
});
|
||||
if (responseClosed || (res.destroyed === true && res.writableEnded !== true)) {
|
||||
execution.abort();
|
||||
}
|
||||
await execution.beginProviderExecution();
|
||||
|
||||
if (request.conversation_id != null) {
|
||||
if (typeof request.conversation_id !== 'string') {
|
||||
return sendErrorResponse(
|
||||
|
|
@ -371,7 +395,6 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
}
|
||||
}
|
||||
|
||||
const conversationId = request.conversation_id ?? nanoid();
|
||||
const parentMessageId = request.parent_message_id ?? null;
|
||||
let mcpParentMessageId;
|
||||
if (typeof request.parent_message_id === 'string' && request.parent_message_id.trim() !== '') {
|
||||
|
|
@ -389,7 +412,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
const allowedProviders = new Set(agentsEConfig?.allowedProviders);
|
||||
|
||||
// Create tool loader
|
||||
const loadTools = createToolLoader(abortController.signal);
|
||||
const loadTools = createToolLoader(execution.signal);
|
||||
|
||||
// Initialize the agent first to check for disableStreaming
|
||||
const endpointOption = {
|
||||
|
|
@ -683,9 +706,6 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
: null;
|
||||
|
||||
const collectedUsage = [];
|
||||
/** @type {Promise<import('librechat-data-provider').TAttachment | null>[]} */
|
||||
const artifactPromises = [];
|
||||
|
||||
const toolEndCallback = createToolEndCallback({ req, res, artifactPromises, streamId: null });
|
||||
|
||||
/* Stable for the turn: the primary prime list is fixed once
|
||||
|
|
@ -706,7 +726,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
requestBody: mcpRequestBody,
|
||||
toolNames,
|
||||
agent: ctx.agent ?? agent,
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
toolRegistry: ctx.toolRegistry,
|
||||
callerCapabilityProjection,
|
||||
backgroundToolNames: ctx.backgroundToolNames,
|
||||
|
|
@ -988,7 +1008,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
runId: responseId,
|
||||
summarizationConfig,
|
||||
appConfig,
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
customHandlers: handlers,
|
||||
requestBody: mcpRequestBody,
|
||||
user: { id: userId },
|
||||
|
|
@ -1012,7 +1032,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
...(userMCPAuthMap != null && { userMCPAuthMap }),
|
||||
},
|
||||
recursionLimit: resolveRecursionLimit(agentsEConfig, agent),
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
streamMode: 'values',
|
||||
version: 'v2',
|
||||
};
|
||||
|
|
@ -1028,28 +1048,30 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
// Record token usage against balance
|
||||
const balanceConfig = getBalanceConfig(appConfig);
|
||||
const transactionsConfig = getTransactionsConfig(appConfig);
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier },
|
||||
bulkWriteOps: { insertMany: db.bulkInsertTransactions, updateBalance: db.updateBalance },
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
});
|
||||
execution.track(
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier },
|
||||
bulkWriteOps: { insertMany: db.bulkInsertTransactions, updateBalance: db.updateBalance },
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[OpenAI API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
}),
|
||||
);
|
||||
|
||||
const usage = buildCompletionUsage(collectedUsage);
|
||||
|
||||
|
|
@ -1060,26 +1082,30 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
res.end();
|
||||
logger.debug(`[OpenAI API] Response ${responseId} completed in ${duration}ms (streaming)`);
|
||||
|
||||
// Wait for artifact processing after response ends (non-blocking)
|
||||
// The HTTP response is complete, while destructive cleanup still waits for artifacts.
|
||||
if (artifactPromises.length > 0) {
|
||||
Promise.all(artifactPromises).catch((artifactError) => {
|
||||
logger.warn(
|
||||
'[OpenAI API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
});
|
||||
execution.track(
|
||||
waitForAgentExecutionWrites(artifactPromises).catch((artifactError) => {
|
||||
logger.warn(
|
||||
'[OpenAI API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
}),
|
||||
);
|
||||
artifactWritesCovered = true;
|
||||
}
|
||||
} else {
|
||||
// For non-streaming, wait for artifacts before sending response
|
||||
if (artifactPromises.length > 0) {
|
||||
try {
|
||||
await Promise.all(artifactPromises);
|
||||
await waitForAgentExecutionWrites(artifactPromises);
|
||||
} catch (artifactError) {
|
||||
logger.warn(
|
||||
'[OpenAI API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
}
|
||||
artifactWritesCovered = true;
|
||||
}
|
||||
|
||||
const response = buildNonStreamingResponse(
|
||||
|
|
@ -1095,6 +1121,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
);
|
||||
}
|
||||
} catch (error) {
|
||||
executionError = error;
|
||||
logger.error('[OpenAI API] Error:', getSafeErrorMetadata(error));
|
||||
const protectionEnabled = hasModelBoundContentProtection(
|
||||
appConfig?.filters,
|
||||
|
|
@ -1129,6 +1156,23 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
|
|||
const errorCode = !protectionEnabled && typeof error?.code === 'string' ? error.code : null;
|
||||
sendErrorResponse(res, statusCode, errorMessage, errorType, errorCode);
|
||||
}
|
||||
} finally {
|
||||
res.off('close', abortOnResponseClose);
|
||||
if (execution) {
|
||||
if (!artifactWritesCovered && artifactPromises.length > 0) {
|
||||
execution.track(
|
||||
waitForAgentExecutionWrites(artifactPromises).catch((artifactError) => {
|
||||
logger.warn(
|
||||
'[OpenAI API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
await execution.settle(executionError).catch((error) => {
|
||||
logger.error('[OpenAI API] Failed to settle execution:', getSafeErrorMetadata(error));
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ const {
|
|||
getLangfuseTraceMessageFields,
|
||||
stripActivityLabelParts,
|
||||
CHILD_THREAD_READ_ONLY_ERROR,
|
||||
enrollAgentExecution,
|
||||
waitForAgentExecutionWrites,
|
||||
} = require('@librechat/api');
|
||||
const {
|
||||
createResponsesToolEndCallback,
|
||||
|
|
@ -573,18 +575,40 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
`[Responses API] Request ${responseId} started for agent ${agentId}, stream: ${isStreaming}`,
|
||||
);
|
||||
|
||||
// Set up abort controller
|
||||
const abortController = new AbortController();
|
||||
|
||||
// Handle client disconnect
|
||||
req.on('close', () => {
|
||||
if (!abortController.signal.aborted) {
|
||||
abortController.abort();
|
||||
const conversationId = request.previous_response_id ?? uuidv4();
|
||||
let execution;
|
||||
let executionError;
|
||||
let responseClosed = res.destroyed === true && res.writableEnded !== true;
|
||||
/** @type {Promise<import('librechat-data-provider').TAttachment | null>[]} */
|
||||
const artifactPromises = [];
|
||||
let artifactWritesCovered = false;
|
||||
const abortOnResponseClose = () => {
|
||||
if (res.writableEnded === true) {
|
||||
return;
|
||||
}
|
||||
responseClosed = true;
|
||||
if (execution && !execution.signal.aborted) {
|
||||
execution.abort();
|
||||
logger.debug('[Responses API] Client disconnected, aborting');
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
res.once('close', abortOnResponseClose);
|
||||
try {
|
||||
execution = await enrollAgentExecution({
|
||||
runId: responseId,
|
||||
userId: principal.userId,
|
||||
conversationId,
|
||||
agentId,
|
||||
protocol: 'responses',
|
||||
/** Conversation delete-all uses the shared owner-admission fence. Remote
|
||||
* execution must observe it after durable enrollment and before provider work. */
|
||||
isPrincipalActive: db.isSubagentOwnerAdmissible,
|
||||
});
|
||||
if (responseClosed || (res.destroyed === true && res.writableEnded !== true)) {
|
||||
execution.abort();
|
||||
}
|
||||
await execution.beginProviderExecution();
|
||||
|
||||
if (request.previous_response_id != null) {
|
||||
if (typeof request.previous_response_id !== 'string') {
|
||||
return sendResponsesErrorResponse(
|
||||
|
|
@ -612,7 +636,6 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
}
|
||||
}
|
||||
|
||||
const conversationId = request.previous_response_id ?? uuidv4();
|
||||
const parentMessageId = null;
|
||||
const mcpRequestBody = createMCPRuntimeRequestBody({ messageId: responseId, conversationId });
|
||||
const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents];
|
||||
|
|
@ -631,7 +654,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
const allowedProviders = new Set(agentsEConfig?.allowedProviders);
|
||||
|
||||
// Create tool loader
|
||||
const loadTools = createToolLoader(abortController.signal);
|
||||
const loadTools = createToolLoader(execution.signal);
|
||||
const skillDbMethods = getSkillDbMethods();
|
||||
|
||||
// Initialize the agent first to check for disableStreaming
|
||||
|
|
@ -1013,8 +1036,6 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
const collectedUsage = [];
|
||||
|
||||
// Artifact promises for processing tool outputs
|
||||
/** @type {Promise<import('librechat-data-provider').TAttachment | null>[]} */
|
||||
const artifactPromises = [];
|
||||
// Use Responses API-specific callback that emits librechat:attachment events
|
||||
const toolEndCallback = createResponsesToolEndCallback({
|
||||
req,
|
||||
|
|
@ -1036,7 +1057,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
requestBody: mcpRequestBody,
|
||||
toolNames,
|
||||
agent: ctx.agent ?? agent,
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
toolRegistry: ctx.toolRegistry,
|
||||
callerCapabilityProjection,
|
||||
backgroundToolNames: ctx.backgroundToolNames,
|
||||
|
|
@ -1100,7 +1121,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
runId: responseId,
|
||||
summarizationConfig,
|
||||
appConfig,
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
customHandlers: handlers,
|
||||
initialSessions,
|
||||
requestBody: mcpRequestBody,
|
||||
|
|
@ -1125,7 +1146,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
requestBody: mcpRequestBody,
|
||||
...(userMCPAuthMap != null && { userMCPAuthMap }),
|
||||
},
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
streamMode: 'values',
|
||||
version: 'v2',
|
||||
};
|
||||
|
|
@ -1141,28 +1162,33 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
// Record token usage against balance
|
||||
const balanceConfig = getBalanceConfig(appConfig);
|
||||
const transactionsConfig = getTransactionsConfig(appConfig);
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier },
|
||||
bulkWriteOps: { insertMany: db.bulkInsertTransactions, updateBalance: db.updateBalance },
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[Responses API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
});
|
||||
execution.track(
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier },
|
||||
bulkWriteOps: {
|
||||
insertMany: db.bulkInsertTransactions,
|
||||
updateBalance: db.updateBalance,
|
||||
},
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[Responses API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
}),
|
||||
);
|
||||
|
||||
const usage = buildResponsesUsage(collectedUsage);
|
||||
|
||||
|
|
@ -1202,14 +1228,17 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
}
|
||||
}
|
||||
|
||||
// Wait for artifact processing after response ends (non-blocking)
|
||||
// The HTTP response is complete, while destructive cleanup still waits for artifacts.
|
||||
if (artifactPromises.length > 0) {
|
||||
Promise.all(artifactPromises).catch((artifactError) => {
|
||||
logger.warn(
|
||||
'[Responses API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
});
|
||||
execution.track(
|
||||
waitForAgentExecutionWrites(artifactPromises).catch((artifactError) => {
|
||||
logger.warn(
|
||||
'[Responses API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
}),
|
||||
);
|
||||
artifactWritesCovered = true;
|
||||
}
|
||||
} else {
|
||||
const aggregatorHandlers = createAggregatorEventHandlers(aggregator);
|
||||
|
|
@ -1217,8 +1246,6 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
// Collect usage for balance tracking
|
||||
const collectedUsage = [];
|
||||
|
||||
/** @type {Promise<import('librechat-data-provider').TAttachment | null>[]} */
|
||||
const artifactPromises = [];
|
||||
const toolEndCallback = createToolEndCallback({ req, res, artifactPromises, streamId: null });
|
||||
|
||||
const toolExecuteOptions = {
|
||||
|
|
@ -1233,7 +1260,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
requestBody: mcpRequestBody,
|
||||
toolNames,
|
||||
agent: ctx.agent ?? agent,
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
toolRegistry: ctx.toolRegistry,
|
||||
callerCapabilityProjection,
|
||||
backgroundToolNames: ctx.backgroundToolNames,
|
||||
|
|
@ -1295,7 +1322,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
runId: responseId,
|
||||
summarizationConfig,
|
||||
appConfig,
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
customHandlers: handlers,
|
||||
initialSessions,
|
||||
requestBody: mcpRequestBody,
|
||||
|
|
@ -1319,7 +1346,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
requestBody: mcpRequestBody,
|
||||
...(userMCPAuthMap != null && { userMCPAuthMap }),
|
||||
},
|
||||
signal: abortController.signal,
|
||||
signal: execution.signal,
|
||||
streamMode: 'values',
|
||||
version: 'v2',
|
||||
};
|
||||
|
|
@ -1335,38 +1362,44 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
// Record token usage against balance
|
||||
const balanceConfig = getBalanceConfig(appConfig);
|
||||
const transactionsConfig = getTransactionsConfig(appConfig);
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier },
|
||||
bulkWriteOps: { insertMany: db.bulkInsertTransactions, updateBalance: db.updateBalance },
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[Responses API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
});
|
||||
execution.track(
|
||||
recordCollectedUsage(
|
||||
{
|
||||
spendTokens: db.spendTokens,
|
||||
spendStructuredTokens: db.spendStructuredTokens,
|
||||
pricing: { getMultiplier: db.getMultiplier, getCacheMultiplier: db.getCacheMultiplier },
|
||||
bulkWriteOps: {
|
||||
insertMany: db.bulkInsertTransactions,
|
||||
updateBalance: db.updateBalance,
|
||||
},
|
||||
},
|
||||
{
|
||||
user: userId,
|
||||
conversationId,
|
||||
collectedUsage,
|
||||
context: 'message',
|
||||
messageId: responseId,
|
||||
balance: balanceConfig,
|
||||
transactions: transactionsConfig,
|
||||
model: primaryConfig.model || agent.model_parameters?.model,
|
||||
endpointTokenConfig: primaryConfig.endpointTokenConfig,
|
||||
resolveEndpointTokenConfig,
|
||||
},
|
||||
).catch((err) => {
|
||||
logger.error('[Responses API] Error recording usage:', getSafeErrorMetadata(err));
|
||||
}),
|
||||
);
|
||||
|
||||
if (artifactPromises.length > 0) {
|
||||
try {
|
||||
await Promise.all(artifactPromises);
|
||||
await waitForAgentExecutionWrites(artifactPromises);
|
||||
} catch (artifactError) {
|
||||
logger.warn(
|
||||
'[Responses API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
}
|
||||
artifactWritesCovered = true;
|
||||
}
|
||||
|
||||
const response = buildAggregatedResponse(
|
||||
|
|
@ -1407,6 +1440,7 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
);
|
||||
}
|
||||
} catch (error) {
|
||||
executionError = error;
|
||||
logger.error('[Responses API] Error:', getSafeErrorMetadata(error));
|
||||
const protectionEnabled = hasModelBoundContentProtection(
|
||||
appConfig?.filters,
|
||||
|
|
@ -1443,6 +1477,23 @@ const executeResponse = async (envelope, { req, res }) => {
|
|||
sendResponsesErrorResponse(res, statusCode, errorMessage, errorType, errorCode);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
res.off('close', abortOnResponseClose);
|
||||
if (execution) {
|
||||
if (!artifactWritesCovered && artifactPromises.length > 0) {
|
||||
execution.track(
|
||||
waitForAgentExecutionWrites(artifactPromises).catch((artifactError) => {
|
||||
logger.warn(
|
||||
'[Responses API] Error processing artifacts:',
|
||||
getSafeErrorMetadata(artifactError),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
await execution.settle(executionError).catch((error) => {
|
||||
logger.error('[Responses API] Failed to settle execution:', getSafeErrorMetadata(error));
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ const archiveAllHandler = jest.fn();
|
|||
const generationJobManager = {
|
||||
getJob: jest.fn().mockResolvedValue(null),
|
||||
abortJob: jest.fn().mockResolvedValue({ success: true }),
|
||||
getCleanupBlockingJobIdsForUser: jest.fn().mockResolvedValue([]),
|
||||
getCleanupBlockingJobIdsForConversations: jest.fn().mockResolvedValue([]),
|
||||
};
|
||||
const subagentActivityHandlerInputs = [];
|
||||
const moderatedTexts = [];
|
||||
|
|
|
|||
|
|
@ -109,6 +109,8 @@ describe('Convos Routes', () => {
|
|||
moderatedTexts.length = 0;
|
||||
generationJobManager.getJob.mockResolvedValue(null);
|
||||
generationJobManager.abortJob.mockResolvedValue({ success: true });
|
||||
generationJobManager.getCleanupBlockingJobIdsForUser.mockResolvedValue([]);
|
||||
generationJobManager.getCleanupBlockingJobIdsForConversations.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it('binds the activity subscription adapter to the subagent task store', () => {
|
||||
|
|
@ -406,6 +408,9 @@ describe('Convos Routes', () => {
|
|||
|
||||
it('drains a paused event actor after owner-wide deletion removes its conversation', async () => {
|
||||
const createdAt = Date.now();
|
||||
generationJobManager.getCleanupBlockingJobIdsForUser.mockResolvedValue([
|
||||
'paused-event-child',
|
||||
]);
|
||||
deleteConvos.mockResolvedValue({
|
||||
deletedCount: 1,
|
||||
conversationIds: ['paused-event-child'],
|
||||
|
|
@ -429,6 +434,108 @@ describe('Convos Routes', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('drains every owner remote run before selecting the delete-all snapshot', async () => {
|
||||
const createdAt = Date.now();
|
||||
generationJobManager.getCleanupBlockingJobIdsForUser.mockResolvedValue([
|
||||
'resp-new-conversation',
|
||||
]);
|
||||
generationJobManager.getJob.mockImplementation(async (streamId) =>
|
||||
streamId === 'resp-new-conversation'
|
||||
? {
|
||||
conversationId: 'conversation-not-yet-persisted',
|
||||
metadata: { userId: 'test-user-123' },
|
||||
status: 'running',
|
||||
createdAt,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
deleteConvos.mockResolvedValue({
|
||||
deletedCount: 1,
|
||||
conversationIds: ['conversation-not-yet-persisted'],
|
||||
});
|
||||
|
||||
const response = await request(app).delete('/api/convos/all');
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.getCleanupBlockingJobIdsForUser).toHaveBeenCalledWith(
|
||||
'test-user-123',
|
||||
undefined,
|
||||
);
|
||||
expect(generationJobManager.abortJob).toHaveBeenCalledWith('resp-new-conversation', {
|
||||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
expect(generationJobManager.abortJob.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
deleteConvos.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('re-drains and removes persistence created during a recovered owner-fence gap', async () => {
|
||||
const createdAt = Date.now();
|
||||
generationJobManager.getCleanupBlockingJobIdsForUser
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValueOnce(['resp-gap']);
|
||||
generationJobManager.getJob.mockImplementation(async (streamId) =>
|
||||
streamId === 'resp-gap'
|
||||
? { metadata: { userId: 'test-user-123' }, status: 'running', createdAt }
|
||||
: null,
|
||||
);
|
||||
deleteConvos
|
||||
.mockResolvedValueOnce({ deletedCount: 1, conversationIds: ['original'] })
|
||||
.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
expect(options).toEqual(
|
||||
expect.objectContaining({ allowEmpty: true, beforeDelete: expect.any(Function) }),
|
||||
);
|
||||
await options.beforeDelete(['gap-conversation']);
|
||||
return { deletedCount: 1, conversationIds: ['gap-conversation'] };
|
||||
});
|
||||
subagentThreadStore.withOwnerDeletionFence.mockImplementationOnce(
|
||||
async (_userId, _tenantId, deletion, recover) => {
|
||||
const result = await deletion();
|
||||
await recover();
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
||||
const response = await request(app).delete('/api/convos/all');
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.abortJob).toHaveBeenCalledWith('resp-gap', {
|
||||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
expect(deleteConvos).toHaveBeenCalledTimes(2);
|
||||
expect(deleteMessages).toHaveBeenCalledWith({ user: 'test-user-123' });
|
||||
expect(deleteAgentCheckpoints.mock.calls.map((call) => call[0])).toEqual([
|
||||
['original'],
|
||||
['gap-conversation'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('retries owner message cleanup after conversations were already removed', async () => {
|
||||
deleteConvos
|
||||
.mockResolvedValueOnce({ deletedCount: 1, conversationIds: ['original'] })
|
||||
.mockResolvedValueOnce({ deletedCount: 0, conversationIds: [] });
|
||||
deleteMessages
|
||||
.mockRejectedValueOnce(new Error('message database unavailable'))
|
||||
.mockResolvedValueOnce({ deletedCount: 1 });
|
||||
|
||||
const first = await request(app).delete('/api/convos/all');
|
||||
const retry = await request(app).delete('/api/convos/all');
|
||||
|
||||
expect(first.status).toBe(500);
|
||||
expect(retry.status).toBe(201);
|
||||
expect(deleteAgentCheckpoints.mock.calls[0][0]).toEqual(['original']);
|
||||
expect(deleteConvos).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'test-user-123',
|
||||
{},
|
||||
expect.objectContaining({ allowEmpty: true }),
|
||||
);
|
||||
expect(deleteMessages).toHaveBeenCalledTimes(2);
|
||||
expect(deleteMessages).toHaveBeenLastCalledWith({ user: 'test-user-123' });
|
||||
});
|
||||
|
||||
it('should delete all conversations, tool calls, and shared links for a user', async () => {
|
||||
const mockDbResponse = {
|
||||
deletedCount: 5,
|
||||
|
|
@ -636,6 +743,36 @@ describe('Convos Routes', () => {
|
|||
{},
|
||||
expect.objectContaining({ beforeDelete: expect.any(Function) }),
|
||||
);
|
||||
expect(subagentThreadStore.withOwnerDeletionFence.mock.calls[0][3]).toEqual(
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it('drains owner remote runs before the empty-filter deletion snapshot', async () => {
|
||||
const createdAt = Date.now();
|
||||
generationJobManager.getCleanupBlockingJobIdsForUser.mockResolvedValue(['resp-new']);
|
||||
generationJobManager.getJob.mockImplementation(async (streamId) =>
|
||||
streamId === 'resp-new'
|
||||
? { metadata: { userId: 'test-user-123' }, status: 'running', createdAt }
|
||||
: null,
|
||||
);
|
||||
deleteConvos.mockResolvedValue({
|
||||
deletedCount: 1,
|
||||
conversationIds: ['new-conversation'],
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { thread_id: 'thread-abc' } });
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.abortJob).toHaveBeenCalledWith('resp-new', {
|
||||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
expect(generationJobManager.abortJob.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
deleteConvos.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('drains a paused event actor after an empty-filter deletion removes it', async () => {
|
||||
|
|
@ -666,9 +803,12 @@ describe('Convos Routes', () => {
|
|||
});
|
||||
|
||||
it('fails closed before checkpoint pruning when generation lookup stays unavailable', async () => {
|
||||
deleteConvos.mockResolvedValue({
|
||||
deletedCount: 1,
|
||||
conversationIds: ['paused-event-child'],
|
||||
deleteConvos.mockImplementation(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['paused-event-child']);
|
||||
return {
|
||||
deletedCount: 1,
|
||||
conversationIds: ['paused-event-child'],
|
||||
};
|
||||
});
|
||||
generationJobManager.getJob.mockRejectedValue(new Error('generation store unavailable'));
|
||||
|
||||
|
|
@ -743,15 +883,172 @@ describe('Convos Routes', () => {
|
|||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
expect(deleteConvos).toHaveBeenNthCalledWith(2, 'test-user-123', {
|
||||
conversationId: { $in: ['parent-conversation', 'child-conversation'] },
|
||||
});
|
||||
expect(deleteConvos).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'test-user-123',
|
||||
{ conversationId: { $in: ['parent-conversation', 'child-conversation'] } },
|
||||
{ allowEmpty: true },
|
||||
);
|
||||
expect(deleteMessages).toHaveBeenCalledWith({
|
||||
user: 'test-user-123',
|
||||
conversationId: { $in: ['parent-conversation', 'child-conversation'] },
|
||||
});
|
||||
});
|
||||
|
||||
it('drains response-id runs indexed under a deleted conversation', async () => {
|
||||
const createdAt = Date.now();
|
||||
generationJobManager.getCleanupBlockingJobIdsForConversations.mockResolvedValue([
|
||||
'resp_remote-run',
|
||||
]);
|
||||
generationJobManager.getJob.mockImplementation(async (streamId) =>
|
||||
streamId === 'resp_remote-run'
|
||||
? { metadata: { userId: 'test-user-123' }, status: 'running', createdAt }
|
||||
: null,
|
||||
);
|
||||
deleteConvos.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['conversation-1']);
|
||||
return { deletedCount: 1, conversationIds: ['conversation-1'] };
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { conversationId: 'conversation-1' } });
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.getCleanupBlockingJobIdsForConversations).toHaveBeenCalledWith(
|
||||
'test-user-123',
|
||||
['conversation-1'],
|
||||
undefined,
|
||||
);
|
||||
expect(generationJobManager.abortJob).toHaveBeenCalledWith('resp_remote-run', {
|
||||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('waits for terminal response-id runs whose provider writes are undrained', async () => {
|
||||
const createdAt = Date.now();
|
||||
generationJobManager.getCleanupBlockingJobIdsForConversations.mockResolvedValue([
|
||||
'resp_terminal-run',
|
||||
]);
|
||||
generationJobManager.getJob.mockImplementation(async (streamId) =>
|
||||
streamId === 'resp_terminal-run'
|
||||
? {
|
||||
metadata: { userId: 'test-user-123', providerDrained: false },
|
||||
status: 'complete',
|
||||
createdAt,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
deleteConvos.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['conversation-1']);
|
||||
return { deletedCount: 1, conversationIds: ['conversation-1'] };
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { conversationId: 'conversation-1' } });
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.abortJob).toHaveBeenCalledWith('resp_terminal-run', {
|
||||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('catches a response-id run admitted after the pre-delete snapshot', async () => {
|
||||
const createdAt = Date.now();
|
||||
let deletionCommitted = false;
|
||||
generationJobManager.getCleanupBlockingJobIdsForConversations.mockImplementation(async () =>
|
||||
deletionCommitted ? ['resp_late-run'] : [],
|
||||
);
|
||||
generationJobManager.getJob.mockImplementation(async (streamId) =>
|
||||
streamId === 'resp_late-run'
|
||||
? { metadata: { userId: 'test-user-123' }, status: 'running', createdAt }
|
||||
: null,
|
||||
);
|
||||
deleteConvos.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['conversation-1']);
|
||||
deletionCommitted = true;
|
||||
return { deletedCount: 1, conversationIds: ['conversation-1'] };
|
||||
});
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { conversationId: 'conversation-1' } });
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.getCleanupBlockingJobIdsForConversations).toHaveBeenCalledTimes(
|
||||
2,
|
||||
);
|
||||
expect(generationJobManager.abortJob).toHaveBeenCalledWith('resp_late-run', {
|
||||
expectedCreatedAt: createdAt,
|
||||
awaitProviderDrain: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('removes persistence even when a racing response drains before post-delete discovery', async () => {
|
||||
deleteConvos
|
||||
.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['conversation-1']);
|
||||
return { deletedCount: 1, conversationIds: ['conversation-1'] };
|
||||
})
|
||||
.mockResolvedValueOnce({ deletedCount: 1, conversationIds: ['conversation-1'] });
|
||||
generationJobManager.getCleanupBlockingJobIdsForConversations.mockResolvedValue([]);
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { conversationId: 'conversation-1' } });
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(generationJobManager.abortJob).not.toHaveBeenCalled();
|
||||
expect(deleteConvos).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'test-user-123',
|
||||
{ conversationId: { $in: ['conversation-1'] } },
|
||||
{ allowEmpty: true },
|
||||
);
|
||||
expect(deleteMessages).toHaveBeenCalledWith({
|
||||
user: 'test-user-123',
|
||||
conversationId: { $in: ['conversation-1'] },
|
||||
});
|
||||
});
|
||||
|
||||
it('fails closed when the idempotent remnant sweep hits a storage failure', async () => {
|
||||
deleteConvos
|
||||
.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['conversation-1']);
|
||||
return { deletedCount: 1, conversationIds: ['conversation-1'] };
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('remnant database unavailable'));
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { conversationId: 'conversation-1' } });
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
expect(response.text).toBe('Error clearing conversations');
|
||||
expect(deleteAgentCheckpoints).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails closed when remnant message cleanup is unavailable', async () => {
|
||||
deleteConvos
|
||||
.mockImplementationOnce(async (_userId, _filter, options) => {
|
||||
await options.beforeDelete(['conversation-1']);
|
||||
return { deletedCount: 1, conversationIds: ['conversation-1'] };
|
||||
})
|
||||
.mockResolvedValueOnce({ deletedCount: 0, conversationIds: [] });
|
||||
deleteMessages.mockRejectedValueOnce(new Error('message database unavailable'));
|
||||
|
||||
const response = await request(app)
|
||||
.delete('/api/convos')
|
||||
.send({ arg: { conversationId: 'conversation-1' } });
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
expect(response.text).toBe('Error clearing conversations');
|
||||
expect(deleteAgentCheckpoints).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not prune generation persistence when provider stop is unconfirmed', async () => {
|
||||
const createdAt = Date.now();
|
||||
let deletionCommitted = false;
|
||||
|
|
@ -1010,6 +1307,7 @@ describe('Convos Routes', () => {
|
|||
});
|
||||
|
||||
expect(executionOrder).toEqual([
|
||||
'deleteConvos',
|
||||
'deleteConvos',
|
||||
'deleteToolCalls',
|
||||
'deleteConvoSharedLinksWithCleanup',
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ const db = require('~/models');
|
|||
const apiKeyMiddleware = createRequireApiKeyAuth({
|
||||
validateAgentApiKey: db.validateAgentApiKey,
|
||||
findUser: db.findUser,
|
||||
isPrincipalActive: db.isAgentTriggerPrincipalActive,
|
||||
});
|
||||
|
||||
const requireRemoteAgentAuth = createRemoteAgentAuth({
|
||||
|
|
@ -21,6 +22,7 @@ const requireRemoteAgentAuth = createRemoteAgentAuth({
|
|||
findUser: db.findUser,
|
||||
getRolesByNames: db.findRolesByNames,
|
||||
updateUser: db.updateUser,
|
||||
isPrincipalActive: db.isAgentTriggerPrincipalActive,
|
||||
getAppConfig,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -275,10 +275,29 @@ async function retryPostDeleteCancellation(cancellationPlan, deletedConversation
|
|||
}
|
||||
|
||||
/** Confirms every exact generation is stopped before its conversation wave is removed. */
|
||||
async function confirmAgentGenerationsDrained(userId, conversationIds, leaseTaskIds = []) {
|
||||
async function confirmAgentGenerationsDrained(
|
||||
userId,
|
||||
conversationIds,
|
||||
leaseTaskIds = [],
|
||||
tenantId,
|
||||
ownerWide = false,
|
||||
) {
|
||||
let foundActiveGeneration = false;
|
||||
const drainErrors = [];
|
||||
const generationIds = [...new Set([...conversationIds, ...leaseTaskIds])];
|
||||
let conversationRunIds;
|
||||
try {
|
||||
conversationRunIds = ownerWide
|
||||
? await GenerationJobManager.getCleanupBlockingJobIdsForUser(userId, tenantId)
|
||||
: await GenerationJobManager.getCleanupBlockingJobIdsForConversations(
|
||||
userId,
|
||||
conversationIds,
|
||||
tenantId,
|
||||
);
|
||||
} catch (error) {
|
||||
logger.warn('Conversation generation index lookup failed', error);
|
||||
throw new Error('Conversation generations could not be confirmed drained.');
|
||||
}
|
||||
const generationIds = [...new Set([...conversationIds, ...leaseTaskIds, ...conversationRunIds])];
|
||||
await Promise.all(
|
||||
generationIds.map(async (conversationId) => {
|
||||
let job;
|
||||
|
|
@ -296,6 +315,7 @@ async function confirmAgentGenerationsDrained(userId, conversationIds, leaseTask
|
|||
const needsDrain =
|
||||
job.status === 'running' ||
|
||||
job.status === 'requires_action' ||
|
||||
job.metadata?.providerDrained === false ||
|
||||
job.metadata?.terminalPersistencePending === true;
|
||||
if (!needsDrain) return;
|
||||
foundActiveGeneration = true;
|
||||
|
|
@ -339,25 +359,53 @@ async function confirmAgentGenerationsDrained(userId, conversationIds, leaseTask
|
|||
return true;
|
||||
}
|
||||
|
||||
/** Stops event-bound child generations on their owning replica and then removes
|
||||
* persistence that raced the first conversation cascade. */
|
||||
async function drainDeletedAgentGenerations(userId, conversationIds, leaseTaskIds = []) {
|
||||
const foundActiveGeneration = await confirmAgentGenerationsDrained(
|
||||
/** Repeats generation discovery after the conversation wave is gone, then always
|
||||
* removes remnants for that immutable deletion set. A remote run may settle and
|
||||
* leave the cleanup index between persisting and this lookup; absence from the
|
||||
* index is therefore not evidence that the second persistence sweep is unnecessary. */
|
||||
async function drainDeletedAgentGenerations(userId, conversationIds, leaseTaskIds = [], tenantId) {
|
||||
await confirmAgentGenerationsDrained(userId, conversationIds, leaseTaskIds, tenantId);
|
||||
await db.deleteConvos(userId, { conversationId: { $in: conversationIds } }, { allowEmpty: true });
|
||||
await db.deleteMessages({ user: userId, conversationId: { $in: conversationIds } });
|
||||
}
|
||||
|
||||
/** Orders every owner-scoped agent execution against a delete-all persistence
|
||||
* snapshot. The recovery callback repeats the non-subagent drain if the durable
|
||||
* fence ever lapses and must be reacquired after deletion has started. */
|
||||
async function withAgentOwnerDeletionFence(userId, tenantId, deletion, recoverPersistence) {
|
||||
const drainRemoteRuns = () => confirmAgentGenerationsDrained(userId, [], [], tenantId, true);
|
||||
let recoveryConversationIds = [];
|
||||
const result = await subagentThreadTaskStore.withOwnerDeletionFence(
|
||||
userId,
|
||||
conversationIds,
|
||||
leaseTaskIds,
|
||||
tenantId,
|
||||
async () => {
|
||||
await drainRemoteRuns();
|
||||
return deletion();
|
||||
},
|
||||
async () => {
|
||||
await drainRemoteRuns();
|
||||
/** Runs only after the fence was restored. No new provider may enter while
|
||||
* persistence created during the gap is removed idempotently. */
|
||||
const recovery = await recoverPersistence();
|
||||
recoveryConversationIds = recovery.conversationIds ?? [];
|
||||
},
|
||||
);
|
||||
if (!foundActiveGeneration) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await db.deleteConvos(userId, { conversationId: { $in: conversationIds } });
|
||||
} catch {
|
||||
// Expected when no generation raced the first cascade.
|
||||
}
|
||||
await db
|
||||
.deleteMessages({ user: userId, conversationId: { $in: conversationIds } })
|
||||
.catch((error) => logger.warn('Deleted child message remnant cleanup failed', error));
|
||||
return { result, recoveryConversationIds };
|
||||
}
|
||||
|
||||
async function deleteOwnerConversationPersistence(userId, filter, tenantId, checkpointer) {
|
||||
const result = await db.deleteConvos(userId, filter, {
|
||||
allowEmpty: true,
|
||||
beforeDelete: (conversationIds) =>
|
||||
confirmAgentGenerationsDrained(userId, conversationIds, [], tenantId),
|
||||
});
|
||||
/** Consume the deletion receipt before the fallible message sweep. A retry after
|
||||
* conversations are gone cannot reconstruct these checkpoint identities. */
|
||||
await deleteAgentCheckpoints(result.conversationIds ?? [], checkpointer);
|
||||
/** Always runs, including an empty conversation retry, so an interrupted writer
|
||||
* that persisted messages first cannot make its cleanup permanently unreachable. */
|
||||
await db.deleteMessages({ user: userId });
|
||||
return result;
|
||||
}
|
||||
|
||||
router.delete('/', configMiddleware, async (req, res) => {
|
||||
|
|
@ -396,8 +444,10 @@ router.delete('/', configMiddleware, async (req, res) => {
|
|||
typeof req.user.tenantId === 'string' && req.user.tenantId !== ''
|
||||
? req.user.tenantId
|
||||
: undefined;
|
||||
const checkpointer = req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer;
|
||||
let cancellationPlan;
|
||||
let dbResponse;
|
||||
let recoveryConversationIds = [];
|
||||
if (filter.conversationId) {
|
||||
/** Resolve the targets while the conversations still exist: the second pass
|
||||
* runs after their rows are gone and can only reach registered owners. */
|
||||
|
|
@ -409,20 +459,26 @@ router.delete('/', configMiddleware, async (req, res) => {
|
|||
await subagentThreadTaskStore.cancelPlan(cancellationPlan);
|
||||
dbResponse = await db.deleteConvos(req.user.id, filter, {
|
||||
beforeDelete: (conversationIds) =>
|
||||
confirmAgentGenerationsDrained(req.user.id, conversationIds),
|
||||
confirmAgentGenerationsDrained(req.user.id, conversationIds, [], tenantId),
|
||||
});
|
||||
} else {
|
||||
/** An empty filter deletes every conversation this owner has, so it runs behind
|
||||
* the same admission fence as `DELETE /all` rather than a bare drain. */
|
||||
dbResponse = await subagentThreadTaskStore.withOwnerDeletionFence(req.user.id, tenantId, () =>
|
||||
db.deleteConvos(req.user.id, filter, {
|
||||
beforeDelete: (conversationIds) =>
|
||||
confirmAgentGenerationsDrained(req.user.id, conversationIds),
|
||||
}),
|
||||
const fencedDeletion = await withAgentOwnerDeletionFence(
|
||||
req.user.id,
|
||||
tenantId,
|
||||
() => deleteOwnerConversationPersistence(req.user.id, filter, tenantId, checkpointer),
|
||||
() => deleteOwnerConversationPersistence(req.user.id, filter, tenantId, checkpointer),
|
||||
);
|
||||
dbResponse = fencedDeletion.result;
|
||||
recoveryConversationIds = fencedDeletion.recoveryConversationIds;
|
||||
}
|
||||
const deletedConversationIds =
|
||||
dbResponse.conversationIds ?? (filter.conversationId ? [filter.conversationId] : []);
|
||||
const deletedConversationIds = [
|
||||
...new Set([
|
||||
...(dbResponse.conversationIds ?? (filter.conversationId ? [filter.conversationId] : [])),
|
||||
...recoveryConversationIds,
|
||||
]),
|
||||
];
|
||||
/** Root deletion closes new child admission. Replay the plan to catch a task
|
||||
* admitted after the first pass but before that fence, extended with the cascade
|
||||
* this deletion reported. */
|
||||
|
|
@ -441,20 +497,20 @@ router.delete('/', configMiddleware, async (req, res) => {
|
|||
deletedConversationIds.includes(lease.conversationId),
|
||||
)
|
||||
.map((lease) => lease.taskId),
|
||||
tenantId,
|
||||
);
|
||||
} else if (deletedConversationIds.length > 0) {
|
||||
/** Owner-wide deletion drains lease-backed tasks before the cascade, but a
|
||||
* requires_action event actor has intentionally released its lease. Its durable
|
||||
* generation is still addressable by the deleted conversation id and must be
|
||||
* terminalized before its checkpoint is pruned. */
|
||||
await drainDeletedAgentGenerations(req.user.id, deletedConversationIds);
|
||||
await drainDeletedAgentGenerations(req.user.id, deletedConversationIds, [], tenantId);
|
||||
}
|
||||
// HITL: prune the deleted conversations' durable checkpoints — a paused run's
|
||||
// checkpoint would otherwise persist until the Mongo TTL. Never throws.
|
||||
await deleteAgentCheckpoints(
|
||||
deletedConversationIds,
|
||||
req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
|
||||
);
|
||||
if (filter.conversationId) {
|
||||
await deleteAgentCheckpoints(deletedConversationIds, checkpointer);
|
||||
}
|
||||
if (filter.conversationId) {
|
||||
await Promise.all(deletedConversationIds.map((id) => db.deleteToolCalls(req.user.id, id)));
|
||||
await Promise.all(
|
||||
|
|
@ -474,28 +530,17 @@ router.delete('/all', configMiddleware, async (req, res) => {
|
|||
typeof req.user.tenantId === 'string' && req.user.tenantId !== ''
|
||||
? req.user.tenantId
|
||||
: undefined;
|
||||
const checkpointer = req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer;
|
||||
/** Fences new child admission for this owner, drains the live ones, and deletes
|
||||
* inside that fence: a child admitted on another replica mid-deletion would
|
||||
* otherwise keep running against conversations that no longer exist. */
|
||||
const dbResponse = await subagentThreadTaskStore.withOwnerDeletionFence(
|
||||
const fencedDeletion = await withAgentOwnerDeletionFence(
|
||||
req.user.id,
|
||||
tenantId,
|
||||
() =>
|
||||
db.deleteConvos(
|
||||
req.user.id,
|
||||
{},
|
||||
{
|
||||
beforeDelete: (conversationIds) =>
|
||||
confirmAgentGenerationsDrained(req.user.id, conversationIds),
|
||||
},
|
||||
),
|
||||
);
|
||||
await drainDeletedAgentGenerations(req.user.id, dbResponse.conversationIds ?? []);
|
||||
// HITL: prune ALL the deleted conversations' durable checkpoints in one bulk pass.
|
||||
await deleteAgentCheckpoints(
|
||||
dbResponse.conversationIds,
|
||||
req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
|
||||
() => deleteOwnerConversationPersistence(req.user.id, {}, tenantId, checkpointer),
|
||||
() => deleteOwnerConversationPersistence(req.user.id, {}, tenantId, checkpointer),
|
||||
);
|
||||
const dbResponse = fencedDeletion.result;
|
||||
await db.deleteToolCalls(req.user.id);
|
||||
await deleteAllSharedLinksWithCleanup(req.user.id);
|
||||
res.status(201).json(dbResponse);
|
||||
|
|
|
|||
|
|
@ -62,3 +62,4 @@ export * from './subagentDelivery';
|
|||
export * from './view';
|
||||
export * from './reasoningLabels';
|
||||
export * from './toolValidation';
|
||||
export * from './remote';
|
||||
|
|
|
|||
1
packages/api/src/agents/remote/index.ts
Normal file
1
packages/api/src/agents/remote/index.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export * from './lifecycle';
|
||||
270
packages/api/src/agents/remote/lifecycle.spec.ts
Normal file
270
packages/api/src/agents/remote/lifecycle.spec.ts
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
import type { GenerationJobManagerClass } from '~/stream';
|
||||
import {
|
||||
enrollAgentExecution,
|
||||
AgentExecutionAdmissionError,
|
||||
waitForAgentExecutionWrites,
|
||||
} from './lifecycle';
|
||||
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
|
||||
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
|
||||
import { GenerationJobManagerClass as JobManager } from '~/stream';
|
||||
|
||||
function createManager(): GenerationJobManagerClass {
|
||||
const manager = new JobManager();
|
||||
manager.configure({
|
||||
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }),
|
||||
eventTransport: new InMemoryEventTransport(),
|
||||
cleanupOnComplete: false,
|
||||
});
|
||||
manager.initialize();
|
||||
return manager;
|
||||
}
|
||||
|
||||
function deferred<T>(): {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
} {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function enrollmentParams(runId: string, conversationId = 'conversation-1') {
|
||||
return {
|
||||
runId,
|
||||
userId: 'user-1',
|
||||
conversationId,
|
||||
agentId: 'agent-1',
|
||||
protocol: 'chat.completions' as const,
|
||||
isPrincipalActive: jest.fn().mockResolvedValue(true),
|
||||
};
|
||||
}
|
||||
|
||||
describe('Agent execution enrollment', () => {
|
||||
let manager: GenerationJobManagerClass;
|
||||
|
||||
beforeEach(() => {
|
||||
manager = createManager();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await manager.destroy();
|
||||
});
|
||||
|
||||
it('registers a cleanup-blocking run before provider execution', async () => {
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('chatcmpl-1'), { manager });
|
||||
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([
|
||||
'chatcmpl-1',
|
||||
]);
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-1')).resolves.toMatchObject({
|
||||
userId: 'user-1',
|
||||
conversationId: 'conversation-1',
|
||||
status: 'running',
|
||||
providerDrained: true,
|
||||
agent_id: 'agent-1',
|
||||
endpoint: 'chat.completions',
|
||||
});
|
||||
expect(enrollment.signal.aborted).toBe(false);
|
||||
});
|
||||
|
||||
it('retires a run when account deletion wins the post-registration recheck', async () => {
|
||||
const beginProviderExecution = jest.spyOn(manager, 'beginProviderExecution');
|
||||
|
||||
await expect(
|
||||
enrollAgentExecution(
|
||||
{
|
||||
...enrollmentParams('chatcmpl-deleting'),
|
||||
isPrincipalActive: jest.fn().mockResolvedValue(false),
|
||||
},
|
||||
{ manager },
|
||||
),
|
||||
).rejects.toMatchObject<Partial<AgentExecutionAdmissionError>>({
|
||||
code: 'ACCOUNT_DELETION_IN_PROGRESS',
|
||||
status: 409,
|
||||
});
|
||||
|
||||
expect(beginProviderExecution).not.toHaveBeenCalled();
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toMatchObject({
|
||||
status: 'error',
|
||||
providerDrained: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves principal-check infrastructure failures after retiring the run', async () => {
|
||||
const infrastructureError = new Error('principal store unavailable');
|
||||
|
||||
await expect(
|
||||
enrollAgentExecution(
|
||||
{
|
||||
...enrollmentParams('chatcmpl-principal-error'),
|
||||
isPrincipalActive: jest.fn().mockRejectedValue(infrastructureError),
|
||||
},
|
||||
{ manager },
|
||||
),
|
||||
).rejects.toBe(infrastructureError);
|
||||
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('refuses provider admission when the request aborted during enrollment', async () => {
|
||||
const beginProviderExecution = jest.spyOn(manager, 'beginProviderExecution');
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('chatcmpl-disconnected'), {
|
||||
manager,
|
||||
});
|
||||
enrollment.abort();
|
||||
|
||||
await expect(enrollment.beginProviderExecution()).rejects.toMatchObject({
|
||||
code: 'RUN_REPLACED',
|
||||
});
|
||||
expect(beginProviderExecution).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains drain ownership when provider-start commits but its acknowledgement is lost', async () => {
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('chatcmpl-start-ambiguous'), {
|
||||
manager,
|
||||
});
|
||||
const originalBeginProviderExecution = manager.beginProviderExecution.bind(manager);
|
||||
const beginProviderExecution = jest.spyOn(manager, 'beginProviderExecution');
|
||||
beginProviderExecution.mockImplementationOnce(async (...args) => {
|
||||
const started = await originalBeginProviderExecution(...args);
|
||||
expect(started).toBe(true);
|
||||
throw new Error('provider-start response lost');
|
||||
});
|
||||
|
||||
await expect(enrollment.beginProviderExecution()).rejects.toThrow(
|
||||
'provider-start response lost',
|
||||
);
|
||||
await expect(enrollment.settle(new Error('provider did not start'))).resolves.toBeUndefined();
|
||||
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-start-ambiguous')).resolves.toMatchObject({
|
||||
status: 'error',
|
||||
providerDrained: true,
|
||||
});
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps terminal work cleanup-blocking until every tracked write settles', async () => {
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('chatcmpl-tail'), { manager });
|
||||
const tail = deferred<void>();
|
||||
enrollment.track(tail.promise);
|
||||
await enrollment.beginProviderExecution();
|
||||
|
||||
const settlement = enrollment.settle();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-tail')).resolves.toMatchObject({
|
||||
status: 'complete',
|
||||
providerDrained: false,
|
||||
});
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([
|
||||
'chatcmpl-tail',
|
||||
]);
|
||||
|
||||
tail.resolve();
|
||||
await settlement;
|
||||
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-tail')).resolves.toMatchObject({
|
||||
status: 'complete',
|
||||
providerDrained: true,
|
||||
});
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('retries terminalization after trailing writes when the first store attempt fails', async () => {
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('chatcmpl-terminal-retry'), {
|
||||
manager,
|
||||
});
|
||||
await enrollment.beginProviderExecution();
|
||||
const completeJob = jest
|
||||
.spyOn(manager, 'completeJob')
|
||||
.mockRejectedValueOnce(new Error('terminal store unavailable'));
|
||||
|
||||
await expect(enrollment.settle()).resolves.toBeUndefined();
|
||||
|
||||
expect(completeJob).toHaveBeenCalledTimes(2);
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-terminal-retry')).resolves.toMatchObject({
|
||||
status: 'complete',
|
||||
providerDrained: true,
|
||||
});
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('does not mark the provider drained while terminalization remains unavailable', async () => {
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('chatcmpl-terminal-outage'), {
|
||||
manager,
|
||||
});
|
||||
await enrollment.beginProviderExecution();
|
||||
jest
|
||||
.spyOn(manager, 'completeJob')
|
||||
.mockRejectedValueOnce(new Error('terminal store unavailable'))
|
||||
.mockRejectedValueOnce(new Error('terminal store still unavailable'));
|
||||
|
||||
await expect(enrollment.settle()).rejects.toThrow('terminal store still unavailable');
|
||||
|
||||
await expect(manager.getJobStore().getJob('chatcmpl-terminal-outage')).resolves.toMatchObject({
|
||||
status: 'running',
|
||||
providerDrained: false,
|
||||
});
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([
|
||||
'chatcmpl-terminal-outage',
|
||||
]);
|
||||
});
|
||||
|
||||
it('lets destructive cleanup abort the canonical signal and wait for trailing writes', async () => {
|
||||
const enrollment = await enrollAgentExecution(enrollmentParams('resp-delete'), { manager });
|
||||
const tail = deferred<void>();
|
||||
enrollment.track(tail.promise);
|
||||
await enrollment.beginProviderExecution();
|
||||
|
||||
const abort = manager.abortJob('resp-delete', { awaitProviderDrain: true });
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(enrollment.signal.aborted).toBe(true);
|
||||
|
||||
let abortSettled = false;
|
||||
void abort.then(() => {
|
||||
abortSettled = true;
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(abortSettled).toBe(false);
|
||||
|
||||
const settlement = enrollment.settle(new Error('aborted'));
|
||||
tail.resolve();
|
||||
await settlement;
|
||||
await expect(abort).resolves.toMatchObject({ success: true });
|
||||
});
|
||||
|
||||
it('keeps concurrent remote runs on one conversation independently enrolled', async () => {
|
||||
await Promise.all([
|
||||
enrollAgentExecution(enrollmentParams('chatcmpl-a', 'conversation-shared'), { manager }),
|
||||
enrollAgentExecution(enrollmentParams('chatcmpl-b', 'conversation-shared'), { manager }),
|
||||
]);
|
||||
|
||||
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual(
|
||||
expect.arrayContaining(['chatcmpl-a', 'chatcmpl-b']),
|
||||
);
|
||||
await expect(
|
||||
manager.getCleanupBlockingJobIdsForConversations('user-1', ['conversation-shared']),
|
||||
).resolves.toEqual(expect.arrayContaining(['chatcmpl-a', 'chatcmpl-b']));
|
||||
});
|
||||
|
||||
it('waits for every trailing write before reporting the first failure', async () => {
|
||||
const failure = new Error('artifact failed');
|
||||
const remaining = deferred<void>();
|
||||
let finished = false;
|
||||
const settlement = waitForAgentExecutionWrites([
|
||||
Promise.reject(failure),
|
||||
remaining.promise,
|
||||
]).finally(() => {
|
||||
finished = true;
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(finished).toBe(false);
|
||||
|
||||
remaining.resolve();
|
||||
await expect(settlement).rejects.toBe(failure);
|
||||
});
|
||||
});
|
||||
220
packages/api/src/agents/remote/lifecycle.ts
Normal file
220
packages/api/src/agents/remote/lifecycle.ts
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
import type { GenerationJobManagerClass } from '~/stream';
|
||||
import type { GenerationJob } from '~/types/stream';
|
||||
import { GenerationJobManager } from '~/stream';
|
||||
|
||||
const ACCOUNT_DELETION_ERROR = 'Account deletion is in progress';
|
||||
const REMOTE_EXECUTION_ERROR = 'Remote agent execution failed';
|
||||
|
||||
export class AgentExecutionAdmissionError extends Error {
|
||||
readonly code: 'ACCOUNT_DELETION_IN_PROGRESS' | 'RUN_REPLACED';
|
||||
readonly status = 409;
|
||||
|
||||
constructor(message: string, code: 'ACCOUNT_DELETION_IN_PROGRESS' | 'RUN_REPLACED') {
|
||||
super(message);
|
||||
this.name = 'AgentExecutionAdmissionError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export interface AgentExecutionEnrollmentParams {
|
||||
runId: string;
|
||||
userId: string;
|
||||
conversationId: string;
|
||||
agentId: string;
|
||||
protocol: 'chat.completions' | 'responses';
|
||||
isPrincipalActive: (userId: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
interface AgentExecutionEnrollmentDeps {
|
||||
manager: GenerationJobManagerClass;
|
||||
}
|
||||
|
||||
export async function waitForAgentExecutionWrites<T>(writes: readonly Promise<T>[]): Promise<void> {
|
||||
const results = await Promise.allSettled(writes);
|
||||
const failure = results.find((result) => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') {
|
||||
throw failure.reason;
|
||||
}
|
||||
}
|
||||
|
||||
export class AgentExecutionEnrollment {
|
||||
readonly runId: string;
|
||||
readonly createdAt: number;
|
||||
readonly signal: AbortSignal;
|
||||
|
||||
private readonly manager: GenerationJobManagerClass;
|
||||
private readonly providerExecutionId: string;
|
||||
private readonly abortController: AbortController;
|
||||
private readonly trailingWrites: Promise<unknown>[] = [];
|
||||
private providerStarted = false;
|
||||
private settlement?: Promise<void>;
|
||||
|
||||
constructor(manager: GenerationJobManagerClass, job: GenerationJob) {
|
||||
const providerExecutionId = job.metadata.providerExecutionId;
|
||||
if (!providerExecutionId) {
|
||||
throw new Error('Agent execution enrollment is missing its provider identity');
|
||||
}
|
||||
this.manager = manager;
|
||||
this.runId = job.streamId;
|
||||
this.createdAt = job.createdAt;
|
||||
this.providerExecutionId = providerExecutionId;
|
||||
this.abortController = job.abortController;
|
||||
this.signal = job.abortController.signal;
|
||||
}
|
||||
|
||||
abort(reason?: unknown): void {
|
||||
this.abortController.abort(reason);
|
||||
}
|
||||
|
||||
track<T>(write: Promise<T>): Promise<T> {
|
||||
if (this.settlement) {
|
||||
throw new Error('Agent execution enrollment is already settling');
|
||||
}
|
||||
this.trailingWrites.push(write);
|
||||
void write.catch(() => undefined);
|
||||
return write;
|
||||
}
|
||||
|
||||
async beginProviderExecution(): Promise<void> {
|
||||
if (this.providerStarted) {
|
||||
throw new Error('Agent provider execution has already started');
|
||||
}
|
||||
if (this.signal.aborted) {
|
||||
throw new AgentExecutionAdmissionError(
|
||||
'Agent execution stopped before provider startup',
|
||||
'RUN_REPLACED',
|
||||
);
|
||||
}
|
||||
let started: boolean;
|
||||
try {
|
||||
started = await this.manager.beginProviderExecution(
|
||||
this.runId,
|
||||
this.createdAt,
|
||||
this.providerExecutionId,
|
||||
);
|
||||
} catch (error) {
|
||||
/** The CAS may have committed before its response was lost. Provider work has
|
||||
* not begun, but settlement still owns acknowledgement of that possible fence. */
|
||||
this.providerStarted = true;
|
||||
throw error;
|
||||
}
|
||||
if (!started) {
|
||||
throw new AgentExecutionAdmissionError(
|
||||
'Agent execution stopped before provider startup',
|
||||
'RUN_REPLACED',
|
||||
);
|
||||
}
|
||||
this.providerStarted = true;
|
||||
if (this.signal.aborted) {
|
||||
throw new AgentExecutionAdmissionError(
|
||||
'Agent execution stopped before provider startup',
|
||||
'RUN_REPLACED',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
settle(error?: unknown): Promise<void> {
|
||||
this.settlement ??= this.settleInternal(error);
|
||||
return this.settlement;
|
||||
}
|
||||
|
||||
private async settleInternal(error?: unknown): Promise<void> {
|
||||
let terminalError: unknown;
|
||||
try {
|
||||
await this.manager.completeJob(
|
||||
this.runId,
|
||||
error == null ? undefined : REMOTE_EXECUTION_ERROR,
|
||||
this.createdAt,
|
||||
);
|
||||
} catch (settleError) {
|
||||
terminalError = settleError;
|
||||
}
|
||||
|
||||
await Promise.allSettled(this.trailingWrites);
|
||||
|
||||
/** A failed terminal write is not allowed to become a drained running job. Retry
|
||||
* after trailing persistence settles; if another terminal owner won meanwhile,
|
||||
* exact-generation readback is the idempotent success receipt. */
|
||||
if (terminalError != null) {
|
||||
try {
|
||||
const completed = await this.manager.completeJob(
|
||||
this.runId,
|
||||
error == null ? undefined : REMOTE_EXECUTION_ERROR,
|
||||
this.createdAt,
|
||||
);
|
||||
if (!completed) {
|
||||
const job = await this.manager.getJob(this.runId);
|
||||
if (
|
||||
job?.createdAt !== this.createdAt ||
|
||||
job.status === 'running' ||
|
||||
job.status === 'requires_action'
|
||||
) {
|
||||
throw terminalError;
|
||||
}
|
||||
}
|
||||
terminalError = undefined;
|
||||
} catch (retryError) {
|
||||
terminalError = retryError;
|
||||
}
|
||||
}
|
||||
|
||||
let drainError: unknown;
|
||||
if (this.providerStarted && terminalError == null) {
|
||||
try {
|
||||
const drained = await this.manager.markProviderExecutionDrained(
|
||||
this.runId,
|
||||
this.createdAt,
|
||||
this.providerExecutionId,
|
||||
);
|
||||
if (!drained) {
|
||||
throw new Error('Agent provider execution drain could not be confirmed');
|
||||
}
|
||||
} catch (error) {
|
||||
drainError = error;
|
||||
}
|
||||
}
|
||||
|
||||
if (terminalError != null) {
|
||||
throw terminalError;
|
||||
}
|
||||
if (drainError != null) {
|
||||
throw drainError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function retireRejectedEnrollment(
|
||||
manager: GenerationJobManagerClass,
|
||||
job: GenerationJob,
|
||||
): Promise<void> {
|
||||
await manager.completeJob(job.streamId, ACCOUNT_DELETION_ERROR, job.createdAt);
|
||||
}
|
||||
|
||||
export async function enrollAgentExecution(
|
||||
params: AgentExecutionEnrollmentParams,
|
||||
deps: AgentExecutionEnrollmentDeps = { manager: GenerationJobManager },
|
||||
): Promise<AgentExecutionEnrollment> {
|
||||
const { runId, userId, conversationId, agentId, protocol, isPrincipalActive } = params;
|
||||
const job = await deps.manager.createJob(runId, userId, conversationId, {
|
||||
initialMetadata: {
|
||||
agent_id: agentId,
|
||||
endpoint: protocol,
|
||||
model: agentId,
|
||||
responseMessageId: runId,
|
||||
},
|
||||
});
|
||||
|
||||
let active = false;
|
||||
try {
|
||||
active = await isPrincipalActive(userId);
|
||||
} catch (error) {
|
||||
await retireRejectedEnrollment(deps.manager, job).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
if (!active) {
|
||||
await retireRejectedEnrollment(deps.manager, job);
|
||||
throw new AgentExecutionAdmissionError(ACCOUNT_DELETION_ERROR, 'ACCOUNT_DELETION_IN_PROGRESS');
|
||||
}
|
||||
|
||||
return new AgentExecutionEnrollment(deps.manager, job);
|
||||
}
|
||||
|
|
@ -4064,10 +4064,17 @@ describe('SubagentThreadTaskStore', () => {
|
|||
const deletionBlocked = new Promise<void>((resolve) => {
|
||||
releaseDeletion = resolve;
|
||||
});
|
||||
const fenced = store.withOwnerDeletionFence(userId, undefined, async () => {
|
||||
await deletionBlocked;
|
||||
return 'deleted';
|
||||
});
|
||||
const fenced = store.withOwnerDeletionFence(
|
||||
userId,
|
||||
undefined,
|
||||
async () => {
|
||||
await deletionBlocked;
|
||||
return 'deleted';
|
||||
},
|
||||
async () => {
|
||||
order.push('remote-drain');
|
||||
},
|
||||
);
|
||||
await renewing;
|
||||
releaseDeletion();
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 20));
|
||||
|
|
@ -4077,7 +4084,7 @@ describe('SubagentThreadTaskStore', () => {
|
|||
await expect(fenced).resolves.toBe('deleted');
|
||||
/** The lost entry is re-taken before the recovery drain and only released after
|
||||
* the in-flight renewal and recovery renewal both settle. */
|
||||
expect(order).toEqual(['fence', 'renew', 'fence', 'renew', 'release']);
|
||||
expect(order).toEqual(['fence', 'renew', 'fence', 'renew', 'remote-drain', 'release']);
|
||||
expect(fenceOwnerAdmission).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -2400,6 +2400,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
|
|||
userId: string,
|
||||
tenantId: string | undefined,
|
||||
deletion: () => Promise<T>,
|
||||
recoverAdditionalOwnerWork?: () => Promise<void>,
|
||||
): Promise<T> {
|
||||
const fenceWindowMs = this.ownerDrainTimeoutMs + this.ownerFenceGraceMs;
|
||||
const token = randomUUID();
|
||||
|
|
@ -2492,6 +2493,10 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
|
|||
fencedUntil = recoveryUntil;
|
||||
fenceLapsed = false;
|
||||
await this.cancelAndDrainForOwner(userId, tenantId);
|
||||
/** The fence is shared by host-owned execution classes that do not use the
|
||||
* subagent lease store. Let the caller re-drain those classes after the same
|
||||
* lapse, while the restored fence still prevents fresh admission. */
|
||||
await recoverAdditionalOwnerWork?.();
|
||||
if (!fenceHeld()) {
|
||||
throw new Error('The subagent admission fence expired while recovering this deletion.');
|
||||
}
|
||||
|
|
|
|||
101
packages/api/src/apiKeys/middleware.auth.spec.ts
Normal file
101
packages/api/src/apiKeys/middleware.auth.spec.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import { Types } from 'mongoose';
|
||||
import type { NextFunction, Response } from 'express';
|
||||
import type { ApiKeyAuthRequest } from './middleware';
|
||||
import { createRequireApiKeyAuth } from './middleware';
|
||||
|
||||
function createResponse(): {
|
||||
res: Response;
|
||||
status: jest.Mock;
|
||||
json: jest.Mock;
|
||||
} {
|
||||
const status = jest.fn();
|
||||
const json = jest.fn();
|
||||
const res = { status, json } as unknown as Response;
|
||||
status.mockReturnValue(res);
|
||||
return { res, status, json };
|
||||
}
|
||||
|
||||
function createRequest(): ApiKeyAuthRequest {
|
||||
return {
|
||||
headers: { authorization: 'Bearer lc-key' },
|
||||
} as ApiKeyAuthRequest;
|
||||
}
|
||||
|
||||
describe('remote Agent API key authentication', () => {
|
||||
it('rejects a valid key while account deletion is fenced', async () => {
|
||||
const userId = new Types.ObjectId();
|
||||
const middleware = createRequireApiKeyAuth({
|
||||
validateAgentApiKey: jest.fn().mockResolvedValue({
|
||||
userId,
|
||||
keyId: new Types.ObjectId(),
|
||||
}),
|
||||
findUser: jest.fn().mockResolvedValue({ _id: userId }),
|
||||
isPrincipalActive: jest.fn().mockResolvedValue(false),
|
||||
});
|
||||
const { res, status, json } = createResponse();
|
||||
const next = jest.fn() as NextFunction;
|
||||
|
||||
await middleware(createRequest(), res, next);
|
||||
|
||||
expect(status).toHaveBeenCalledWith(409);
|
||||
expect(json).toHaveBeenCalledWith({
|
||||
error: {
|
||||
message: 'Account deletion is in progress',
|
||||
type: 'invalid_request_error',
|
||||
code: 'account_deletion_in_progress',
|
||||
},
|
||||
});
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('admits an active principal', async () => {
|
||||
const userId = new Types.ObjectId();
|
||||
const middleware = createRequireApiKeyAuth({
|
||||
validateAgentApiKey: jest.fn().mockResolvedValue({
|
||||
userId,
|
||||
keyId: new Types.ObjectId(),
|
||||
}),
|
||||
findUser: jest.fn().mockResolvedValue({ _id: userId }),
|
||||
isPrincipalActive: jest.fn().mockResolvedValue(true),
|
||||
});
|
||||
const req = createRequest();
|
||||
const { res } = createResponse();
|
||||
const next = jest.fn() as NextFunction;
|
||||
|
||||
await middleware(req, res, next);
|
||||
|
||||
expect(req.user?.id).toBe(userId.toString());
|
||||
expect(next).toHaveBeenCalledWith();
|
||||
});
|
||||
|
||||
it('starts the user and deletion-fence reads together', async () => {
|
||||
const userId = new Types.ObjectId();
|
||||
let resolveUser!: (user: { _id: Types.ObjectId }) => void;
|
||||
const user = new Promise<{ _id: Types.ObjectId }>((resolve) => {
|
||||
resolveUser = resolve;
|
||||
});
|
||||
const findUser = jest.fn().mockReturnValue(user);
|
||||
const isPrincipalActive = jest.fn().mockResolvedValue(true);
|
||||
const middleware = createRequireApiKeyAuth({
|
||||
validateAgentApiKey: jest.fn().mockResolvedValue({
|
||||
userId,
|
||||
keyId: new Types.ObjectId(),
|
||||
}),
|
||||
findUser,
|
||||
isPrincipalActive,
|
||||
});
|
||||
const { res } = createResponse();
|
||||
const next = jest.fn() as NextFunction;
|
||||
|
||||
const authentication = middleware(createRequest(), res, next);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(findUser).toHaveBeenCalledTimes(1);
|
||||
expect(isPrincipalActive).toHaveBeenCalledWith(userId.toString());
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
|
||||
resolveUser({ _id: userId });
|
||||
await authentication;
|
||||
expect(next).toHaveBeenCalledWith();
|
||||
});
|
||||
});
|
||||
|
|
@ -11,6 +11,7 @@ export interface ApiKeyAuthDependencies {
|
|||
keyId: Types.ObjectId;
|
||||
} | null>;
|
||||
findUser: (query: { _id: string | Types.ObjectId }) => Promise<IUser | null>;
|
||||
isPrincipalActive: (userId: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface RemoteAgentAccessDependencies {
|
||||
|
|
@ -80,7 +81,11 @@ export function createRequireApiKeyAuth(deps: ApiKeyAuthDependencies) {
|
|||
});
|
||||
}
|
||||
|
||||
const user = await deps.findUser({ _id: keyValidation.userId });
|
||||
const userId = keyValidation.userId.toString();
|
||||
const [user, principalActive] = await Promise.all([
|
||||
deps.findUser({ _id: keyValidation.userId }),
|
||||
deps.isPrincipalActive(userId),
|
||||
]);
|
||||
|
||||
if (!user) {
|
||||
return res.status(401).json({
|
||||
|
|
@ -93,6 +98,15 @@ export function createRequireApiKeyAuth(deps: ApiKeyAuthDependencies) {
|
|||
}
|
||||
|
||||
user.id = (user._id as Types.ObjectId).toString();
|
||||
if (!principalActive) {
|
||||
return res.status(409).json({
|
||||
error: {
|
||||
message: 'Account deletion is in progress',
|
||||
type: 'invalid_request_error',
|
||||
code: 'account_deletion_in_progress',
|
||||
},
|
||||
});
|
||||
}
|
||||
req.user = user as IUser & { id: string };
|
||||
req.apiKeyId = keyValidation.keyId;
|
||||
|
||||
|
|
|
|||
|
|
@ -228,6 +228,7 @@ function makeDeps(appConfig: AppConfig = makeConfig()) {
|
|||
getRolesByNames: jest.fn(async (roleNames: string[]) =>
|
||||
roleNames.map((roleName) => ({ name: roleName })),
|
||||
),
|
||||
isPrincipalActive: jest.fn().mockResolvedValue(true),
|
||||
getAppConfig: jest.fn().mockResolvedValue(appConfig),
|
||||
apiKeyMiddleware: jest.fn((_req: unknown, _res: unknown, next: () => void) => next()),
|
||||
};
|
||||
|
|
@ -485,6 +486,24 @@ describe('createRemoteAgentAuth', () => {
|
|||
expect(deps.apiKeyMiddleware).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an OIDC principal while account deletion is fenced', async () => {
|
||||
setupOidcMocks({ sub: 'sub123', email: 'agent@test.com', exp: 9999999999 });
|
||||
const deps = makeDeps();
|
||||
deps.isPrincipalActive.mockResolvedValue(false);
|
||||
const req = makeReq({ authorization: `Bearer ${FAKE_TOKEN}` });
|
||||
const { res, status, json } = makeRes();
|
||||
|
||||
await createRemoteAgentAuth(asDeps(deps))(req as Request, res, mockNext);
|
||||
|
||||
expect(status).toHaveBeenCalledWith(409);
|
||||
expect(json).toHaveBeenCalledWith({
|
||||
error: 'Account deletion is in progress',
|
||||
code: 'ACCOUNT_DELETION_IN_PROGRESS',
|
||||
});
|
||||
expect(deps.updateUser).not.toHaveBeenCalled();
|
||||
expect(mockNext).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('restores tenant context from the OIDC user before continuing', async () => {
|
||||
setupOidcMocks({ sub: 'sub123', email: 'agent@test.com' });
|
||||
const deps = makeDeps();
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ export interface RemoteAgentAuthDeps {
|
|||
findUser: UserMethods['findUser'];
|
||||
getRolesByNames: RoleMethods['findRolesByNames'];
|
||||
updateUser: UserMethods['updateUser'];
|
||||
isPrincipalActive: (userId: string) => Promise<boolean>;
|
||||
getAppConfig: (options?: GetAppConfigOptions) => Promise<AppConfig>;
|
||||
}
|
||||
|
||||
|
|
@ -601,6 +602,7 @@ export function createRemoteAgentAuth({
|
|||
findUser,
|
||||
getRolesByNames,
|
||||
updateUser,
|
||||
isPrincipalActive,
|
||||
getAppConfig,
|
||||
}: RemoteAgentAuthDeps): RequestHandler {
|
||||
/**
|
||||
|
|
@ -729,6 +731,14 @@ export function createRemoteAgentAuth({
|
|||
return;
|
||||
}
|
||||
|
||||
if (!(await isPrincipalActive(userResolution.user.id))) {
|
||||
res.status(409).json({
|
||||
error: 'Account deletion is in progress',
|
||||
code: 'ACCOUNT_DELETION_IN_PROGRESS',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await updateResolvedUser(userResolution, updateUser);
|
||||
|
||||
req.user = userResolution.user;
|
||||
|
|
|
|||
|
|
@ -7974,6 +7974,26 @@ class GenerationJobManagerClass {
|
|||
return this.jobStore.getCleanupBlockingJobIdsByUser(userId, tenantId);
|
||||
}
|
||||
|
||||
/** Resolves every cleanup-blocking run attached to any target conversation.
|
||||
* Remote API runs use response IDs as stream identities, so conversation
|
||||
* deletion cannot assume one stream per conversation. */
|
||||
async getCleanupBlockingJobIdsForConversations(
|
||||
userId: string,
|
||||
conversationIds: readonly string[],
|
||||
tenantId?: string,
|
||||
): Promise<string[]> {
|
||||
if (conversationIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const targets = new Set(conversationIds);
|
||||
const streamIds = await this.jobStore.getCleanupBlockingJobIdsByUser(userId, tenantId);
|
||||
const jobs = await Promise.all(streamIds.map((streamId) => this.jobStore.getJob(streamId)));
|
||||
return streamIds.filter((_, index) => {
|
||||
const job = jobs[index];
|
||||
return job != null && job.userId === userId && targets.has(job.conversationId ?? '');
|
||||
});
|
||||
}
|
||||
|
||||
private async finalizeOwnedJobsForShutdown(): Promise<void> {
|
||||
const ownedJobs = [...this.ownedJobs];
|
||||
if (ownedJobs.length === 0) {
|
||||
|
|
|
|||
|
|
@ -1963,6 +1963,29 @@ describe('Conversation Operations', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('supports an idempotent empty recovery sweep without hiding storage failures', async () => {
|
||||
await expect(
|
||||
deleteConvos(
|
||||
'user123',
|
||||
{ conversationId: { $in: ['already-absent'] } },
|
||||
{ allowEmpty: true },
|
||||
),
|
||||
).resolves.toEqual({
|
||||
acknowledged: true,
|
||||
deletedCount: 0,
|
||||
messages: { acknowledged: true, deletedCount: 0 },
|
||||
conversationIds: [],
|
||||
});
|
||||
|
||||
const find = jest.spyOn(Conversation, 'find').mockImplementationOnce(() => {
|
||||
throw new Error('database unavailable');
|
||||
});
|
||||
await expect(deleteConvos('user123', {}, { allowEmpty: true })).rejects.toThrow(
|
||||
'database unavailable',
|
||||
);
|
||||
find.mockRestore();
|
||||
});
|
||||
|
||||
it('should decrement tag counts for a deleted bookmarked conversation', async () => {
|
||||
await ConversationTag.create({ user: 'user123', tag: 'work', count: 2, position: 1 });
|
||||
const convoId = uuidv4();
|
||||
|
|
|
|||
|
|
@ -456,7 +456,10 @@ export interface ConversationMethods {
|
|||
deleteConvos(
|
||||
user: string,
|
||||
filter: FilterQuery<IConversation>,
|
||||
options?: { beforeDelete?: (conversationIds: string[]) => Promise<void> },
|
||||
options?: {
|
||||
beforeDelete?: (conversationIds: string[]) => Promise<void>;
|
||||
allowEmpty?: boolean;
|
||||
},
|
||||
): Promise<DeleteResult & { messages: DeleteResult; conversationIds: string[] }>;
|
||||
archiveAllConvos(user: string): Promise<{ archivedCount: number }>;
|
||||
}
|
||||
|
|
@ -2834,7 +2837,12 @@ export function createConversationMethods(
|
|||
async function deleteConvos(
|
||||
user: string,
|
||||
filter: FilterQuery<IConversation>,
|
||||
options?: { beforeDelete?: (conversationIds: string[]) => Promise<void> },
|
||||
options?: {
|
||||
beforeDelete?: (conversationIds: string[]) => Promise<void>;
|
||||
/** Idempotent destructive-recovery mode. An empty selection is success, while
|
||||
* query, cascade, reconciliation, and deletion failures still propagate. */
|
||||
allowEmpty?: boolean;
|
||||
},
|
||||
) {
|
||||
try {
|
||||
const Conversation = mongoose.models.Conversation as Model<IConversation>;
|
||||
|
|
@ -2881,6 +2889,14 @@ export function createConversationMethods(
|
|||
conversations = descendants;
|
||||
recoveryConversationIds.push(filter.conversationId);
|
||||
} else if (!conversations.length) {
|
||||
if (options?.allowEmpty === true) {
|
||||
return {
|
||||
acknowledged: true,
|
||||
deletedCount: 0,
|
||||
messages: { acknowledged: true, deletedCount: 0 },
|
||||
conversationIds: [],
|
||||
};
|
||||
}
|
||||
throw new Error('Conversation not found or already deleted.');
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue