🗄️ feat: Persist and Expire Background Responses (#17594)

* 🗄️ feat: Persist Background Responses

* fix: unify response retirement and cleanup identity

* fix: preserve immutable conversation identity and retry capability probes

* test: Await Terminal Replay Instead of a Live Subscription

* test: Keep Delayed Queue Admission Fixtures Consistent

* test: Scope Upload Assertions to the Composer Tray

* test: Observe Paused Writes and Scope Editor Controls

* ci: Allow Complete Database Mutation Reports
This commit is contained in:
Dustin Healy 2026-11-10 21:03:56 -08:00 • committed by GitHub
parent b7fa9c1e71
commit 1458e841e4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 3321 additions and 104 deletions

View file

@ -74,7 +74,8 @@ jobs:
needs: [select, verify]
if: needs.select.outputs.hasChanges == 'true'
runs-on: ubuntu-latest
timeout-minutes: 45
# Whole edited database functions can produce thousands of mutation cases.
timeout-minutes: ${{ matrix.check == 'mutation' && 240 || 45 }}
defaults:
run:
working-directory: source

View file

@ -341,7 +341,9 @@ test.describe('core chat loop', () => {
await uploadProviderFile(page, csvFixture);
await expect(page.getByRole('button', { name: filename })).toBeVisible();
await expect(
page.getByTestId('composer-tray').getByRole('button', { name: filename, exact: true }),
).toBeVisible();
const input = page.getByRole('textbox', { name: 'Message input' });
await input.click();

View file

@ -1,4 +1,5 @@
import { expect, test } from '@playwright/test';
import type { TMessage } from 'librechat-data-provider';
import type { Page, Route } from '@playwright/test';
import type { AgentDetail } from './agents.helpers';
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
@ -27,6 +28,7 @@ const messageTurns = (page: Page) => messagesView(page).locator('.message-render
const RECEIPTS_ROUTE = /\/api\/agents\/chat\/queued-turns\?/;
const STATUS_ROUTE = /\/api\/agents\/chat\/status\//;
const MESSAGES_ROUTE = /\/api\/messages\/[^/?]+(?:\?|$)/;
type ReceiptsBody = { queuedTurns?: Array<{ status?: string }> };
type StatusBody = { active?: boolean; resumeState?: { userMessage?: { text?: string } } };
@ -39,8 +41,9 @@ type StatusBody = { active?: boolean; resumeState?: { userMessage?: { text?: str
* only through the receipt projection and the status it re-arms. While held
* back, the receipts stay frozen at their pre-admission snapshot and a status
* that describes the successor reads as inactive, so the next user turn can
* only come from the completion event itself. Every other request, including
* the predecessor's own terminal status, passes through untouched.
* only come from the completion event itself. History also withholds the admitted
* successor and its descendants, so a backfill cannot bypass the hold. Other requests,
* including the predecessor's own terminal status, pass through untouched.
*/
function holdBackSuccessor(page: Page, successorText: string) {
let holding = false;
@ -70,10 +73,39 @@ function holdBackSuccessor(page: Page, successorText: string) {
json: describesSuccessor ? { active: false } : body,
});
};
const messages = async (route: Route) => {
const response = await route.fetch();
if (!holding || response.status() !== 200 || route.request().method() !== 'GET') {
return route.fulfill({ response });
}
const body = (await response.json()) as TMessage[];
const hidden = new Set<string>();
const children = new Map<string, string[]>();
for (const message of body) {
if (message.isCreatedByUser && message.text === successorText) hidden.add(message.messageId);
const parent = message.parentMessageId ?? '';
const siblings = children.get(parent) ?? [];
siblings.push(message.messageId);
children.set(parent, siblings);
}
const pending = [...hidden];
for (const id of pending) {
for (const child of children.get(id) ?? []) {
if (hidden.has(child)) continue;
hidden.add(child);
pending.push(child);
}
}
return route.fulfill({
response,
json: body.filter((message) => !hidden.has(message.messageId)),
});
};
return {
arm: async () => {
await page.route(RECEIPTS_ROUTE, receipts);
await page.route(STATUS_ROUTE, status);
await page.route(MESSAGES_ROUTE, messages);
},
hold: () => {
holding = true;
@ -82,6 +114,7 @@ function holdBackSuccessor(page: Page, successorText: string) {
holding = false;
await page.unroute(RECEIPTS_ROUTE, receipts);
await page.unroute(STATUS_ROUTE, status);
await page.unroute(MESSAGES_ROUTE, messages);
},
};
}
@ -166,13 +199,14 @@ test.describe('server-queued follow-up reveal', () => {
const predecessor = messageTurns(page).nth(3);
await expect(predecessor).toContainText(SLOW_REPLY_LAST_CHUNK, { timeout: 60000 });
const pending = page.getByTestId('pending-turn');
await expect(pending).toBeVisible({ timeout: 5000 });
const queuedTurn = messageTurns(page).nth(4);
await expect(queuedTurn).toContainText(queueText, { timeout: 5000 });
await expect(queuedTurn.locator('.user-turn')).toBeVisible();
/** One visible copy: actions follow the user turn, not a duplicate chip
* below the composer. Other queued turns still use their ordinary rows. */
await expect(row).toHaveCount(0);
const pending = page.getByTestId('pending-turn');
await expect(pending).toContainText('Starting as the next turn');
await expect(pending.getByRole('button', { name: 'Send now' })).toHaveCount(0);
await expect(pending.getByRole('button', { name: 'Remove message' })).toBeVisible();

View file

@ -507,7 +507,7 @@ test.describe('compaction rerun controls', () => {
await page.locator(`[id="edit-${chainedId}"]`).click();
const editor = page.getByTestId('message-text-editor');
await expect(editor).toBeVisible();
await expect(page.getByRole('button', { name: 'Save' })).toBeVisible();
await expect(page.getByRole('button', { name: 'Save', exact: true })).toBeVisible();
await expect(page.getByRole('button', { name: 'Rerun', exact: true })).toHaveCount(0);
await expect(page.getByRole('button', { name: 'Update & rerun' })).toHaveCount(0);
/* ...and the footer says why the action it usually carries is gone. */
@ -545,7 +545,7 @@ test.describe('compaction rerun controls', () => {
await page.locator(`[id="edit-${rootReplyId}"]`).click();
const editor = page.getByTestId('message-text-editor');
await expect(editor).toBeVisible();
await expect(page.getByRole('button', { name: 'Save' })).toBeVisible();
await expect(page.getByRole('button', { name: 'Save', exact: true })).toBeVisible();
await expect(page.getByRole('button', { name: 'Rerun', exact: true })).toHaveCount(0);
await expect(
page.getByText('Rerunning replays the user message before a response', { exact: false }),

View file

@ -41,7 +41,9 @@ async function attachMarkdown(page: Page) {
expect(response.ok()).toBeTruthy();
const file = (await response.json()) as UploadedFile & { filepath?: string };
expect(file.llmDeliveryPath).toBe('provider');
await expect(page.getByRole('button', { name })).toBeVisible({ timeout: 15000 });
await expect(
page.getByTestId('composer-tray').getByRole('button', { name, exact: true }),
).toBeVisible({ timeout: 15000 });
return { name, file };
}

View file

@ -87,7 +87,9 @@ test.describe('Google native media in the context budget', () => {
});
expect(upload.ok()).toBeTruthy();
expect(((await upload.json()) as UploadedFile).llmDeliveryPath).toBe('provider');
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
await expect(
page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }),
).toBeVisible({ timeout: 15000 });
const report = await reportContextTokens(page);
expect(report.provider).toBe('google');
@ -107,7 +109,9 @@ test.describe('Google native media in the context budget', () => {
content: syntheticPdf(),
});
expect(upload.ok()).toBeTruthy();
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
await expect(
page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }),
).toBeVisible({ timeout: 15000 });
const first = await reportContextTokens(page);
expect(first.latest).toBeGreaterThan(ONE_PAGE_TOKENS);

View file

@ -148,7 +148,11 @@ test.describe('shared links', () => {
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
await uploadProviderFile(page, fileFixture);
await expect(page.getByRole('button', { name: fileFixture.name, exact: true })).toBeVisible();
await expect(
page
.getByTestId('composer-tray')
.getByRole('button', { name: fileFixture.name, exact: true }),
).toBeVisible();
const response = await sendMessage(page, userMessage);
expect(response.ok()).toBeTruthy();

View file

@ -125,8 +125,8 @@ test.describe('skill file authoring tools', () => {
await expect(page.getByRole('heading', { level: 1, name: skillName })).toBeVisible();
await expect(page.getByText(EDITED_DESCRIPTION)).toBeVisible();
await page.getByRole('button', { name: 'Delete' }).click();
await page.getByRole('dialog').getByRole('button', { name: 'Delete' }).click();
await page.getByTestId('app-pane').getByRole('button', { name: 'Delete', exact: true }).click();
await page.getByRole('dialog').getByRole('button', { name: 'Delete', exact: true }).click();
await expect(page).toHaveURL((url) => url.pathname === '/skills');
const token = await getAccessToken(page);
await expect.poll(async () => findSkill(page, skillName, token)).toBeNull();

View file

@ -93,7 +93,9 @@ test.describe('unified file upload', () => {
expect(((await response.json()) as UploadedFile).llmDeliveryPath).toBe('provider');
// (a) shows as an attachment chip in the composer before sending.
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
await expect(
page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }),
).toBeVisible({ timeout: 15000 });
// (b) reaches the model input: the mock LLM echoes a pass marker only when the
// provider file is present in the request content (see e2e/setup/fake-model.js).

View file

@ -400,12 +400,12 @@ describe('terminal projection boundary', () => {
delivered.push(event);
},
);
expect(subscription).not.toBeNull();
await new Promise((resolve) => setTimeout(resolve, 250));
// A completed replay may retire its subscription before attachment returns.
await waitFor(() => delivered.length > 0, 'legacy terminal replay');
expect(delivered).toHaveLength(1);
expectSafeTerminalEvent(delivered[0]);
subscription!.unsubscribe();
subscription?.unsubscribe();
} finally {
await replica.destroy();
await replicaSubscriber.quit().catch(() => undefined);

View file

@ -1140,6 +1140,12 @@ const remoteApiAuthSchema = z.object({
const remoteApiSchema = z.object({
auth: remoteApiAuthSchema.optional(),
background: z
.object({
enabled: z.boolean().default(false),
acknowledgementTimeoutMs: z.number().int().min(100).max(60_000).default(10_000),
})
.optional(),
});
const managementClientBindingSchema = z

View file

@ -39,6 +39,7 @@ export * from './roles';
/* types (exports schemas from `./types` as they contain needed in other defs) */
export * from './types';
export * from './types/agents';
export * from './types/responseAccounting';
export * from './types/assistants';
export * from './types/content';
export * from './types/tools';
@ -112,4 +113,6 @@ export * from './types/scheduleConsent';
export * from './approval';
export * from './types/responseReview';
export * from './codeSetup';

View file

@ -0,0 +1,45 @@
/** An application-billing amount is not a provider invoice or list-price estimate. */
export type AccountingBasis = 'provider_reported' | 'application_billing';
export type AccountingModelSource =
| 'provider_reported'
| 'runtime_reported'
| 'configured'
| 'unknown';
export interface ResponseAccountingCall {
provider?: string;
model?: string;
modelSource: AccountingModelSource;
usageType:
| 'message'
| 'summarization'
| 'subagent'
| 'sequential'
| 'activity-label'
| 'activity-phase'
| 'reasoning-label';
inputTokens: number;
outputTokens: number;
cacheReadTokens: number;
cacheWriteTokens: number;
amount?: string;
basis?: AccountingBasis;
reason: 'priced' | 'pricing_unavailable' | 'usage_unavailable' | 'invalid_usage';
}
/** Separate from token usage: partial costs must never appear as a run's total. */
export interface ResponseAccounting {
status: 'available' | 'unavailable';
currency: 'USD';
amount?: string;
basis?: AccountingBasis;
coverage: 'complete' | 'partial' | 'none';
reason:
| 'priced'
| 'pricing_unavailable'
| 'usage_unavailable'
| 'partial_coverage'
| 'mixed_bases'
| 'conflicting_usage';
calls: ResponseAccountingCall[];
}

View file

@ -0,0 +1,55 @@
import { z } from 'zod';
import type { Agents } from './agents';
export const responseReviewDecisionSchema = z
.object({
actionId: z.string().min(1).max(256),
answer: z.string().optional(),
answers: z.record(z.string(), z.string()).optional(),
decisions: z
.array(
z
.object({
tool_call_id: z.string(),
decision: z.enum(['approve', 'reject', 'edit', 'respond']),
editedArguments: z.record(z.string(), z.unknown()).optional(),
responseText: z.string().optional(),
reason: z.string().optional(),
scope: z.enum(['once', 'session', 'always']).optional(),
})
.strict(),
)
.optional(),
})
.strict();
export type ResponseReviewDecision = z.infer<typeof responseReviewDecisionSchema>;
export const responseReviewDecisionReceiptSchema = z.object({
responseId: z.string().min(1),
actionId: z.string().min(1),
intentId: z.string().min(1),
status: z.enum(['queued', 'settled']),
});
export type ResponseReviewDecisionReceipt = z.infer<typeof responseReviewDecisionReceiptSchema>;
export const responseCancelReceiptSchema = z.object({
id: z.string().min(1),
status: z.enum(['queued', 'in_progress', 'completed', 'failed', 'incomplete', 'cancelled']),
});
export type ResponseCancelReceipt = z.infer<typeof responseCancelReceiptSchema>;
/** Owner-authenticated browser view; OAuth URLs must never be delivered to a channel. */
export interface ResponseReviewView {
/** Missing/mismatched execution is not proof that provider work stopped. */
recovery?: 'execution_state_unavailable';
responseId: string;
status: 'queued' | 'in_progress' | 'completed' | 'failed' | 'incomplete' | 'cancelled';
oauth: Array<{
actionId: string;
toolName: string;
authURL: string;
expiresAt: number;
}>;
pendingAction?: Pick<Agents.PendingAction, 'actionId' | 'payload' | 'expiresAt'>;
pendingDecision?: { intentId: string; decision: ResponseReviewDecision };
canSubmitDecision?: boolean;
}

View file

@ -11,6 +11,22 @@ export * from './crypto';
export * from './schema';
export * from './utils';
export { createModels } from './models';
export { createResponseStorage } from './methods/response';
export type {
ResponseOwner,
ResponseStorage,
ResponseCleanupIdentity,
ResponseReservation,
ResponseReservationResult,
ResponseMutationResult,
ResponseReviewCheckpoint,
ResponseResumeIntent,
ResponseResumeReservation,
ResponseResumeResult,
ResponseTerminalReceipt,
StoredResponseRecord,
StoredResponseStatus,
} from './methods/response';
export {
createMethods,
createSkillDeletionPersistenceAdapter,
@ -103,6 +119,7 @@ export {
AUDIT_ACTION_CATEGORY,
} from './types/admin';
export { GENESIS_HASH, PLATFORM_CHAIN_KEY } from './schema/auditLog';
export { default as logger, baseLogFormat } from './config/winston';
export { default as meiliLogger } from './config/meiliLogger';
export { jsonTruncateFormat, redactMessage } from './config/parsers';

View file

@ -20,6 +20,8 @@ import type {
import { ConversationMethods, createConversationMethods } from './conversation';
import { tenantStorage, runAsSystem } from '~/config/tenantContext';
import { createChatProjectMethods } from './chatProject';
import { createResponseStorage } from './response';
import { waitForPause } from './test-helpers';
import { createModels } from '../models';
jest.mock('~/config/winston', () => ({
@ -1141,13 +1143,13 @@ describe('Conversation Operations', () => {
},
);
const archive = saveConvo(
{ userId: 'user123' },
{ conversationId, isArchived: true },
{ preserveUpdatedAt: true, noUpsert: true },
);
try {
const archive = saveConvo(
{ userId: 'user123' },
{ conversationId, isArchived: true },
{ preserveUpdatedAt: true, noUpsert: true },
);
await archiveWriteReached;
await waitForPause(archiveWriteReached, archive);
const unarchived = await saveConvo(
{ userId: 'user123' },
@ -1166,6 +1168,7 @@ describe('Conversation Operations', () => {
} finally {
releaseArchiveWrite();
writeSpy.mockRestore();
await Promise.allSettled([archive]);
}
});
@ -3848,6 +3851,150 @@ describe('Conversation Operations', () => {
});
describe('deleteConvos', () => {
describe('stored responses on standalone Mongo', () => {
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.ResponseScope.deleteMany({});
});
});
async function seedResponse(conversationId: string, tenantId?: string) {
return runAsSystem(() =>
mongoose.models.StoredResponse.create({
userId: 'user123',
tenantId,
conversationId,
responseId: uuidv4(),
requestKey: uuidv4(),
fingerprint: 'request',
agentId: 'agent',
requestJSON: '{"input":"private"}',
responseJSON: '{"output":"private"}',
parentMessageId: 'root',
status: 'completed',
}),
);
}
it('rejects background reservation on standalone Mongo before storing private state', async () => {
const result = await createResponseStorage(mongoose).reserve({
userId: 'user123',
responseId: uuidv4(),
requestKey: uuidv4(),
fingerprint: 'request',
agentId: 'agent',
requestJSON: '{"input":"private"}',
conversationId: uuidv4(),
parentMessageId: 'root',
});
expect(result).toEqual({ ok: false, code: 'response_unavailable' });
expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0);
expect(await mongoose.models.ResponseScope.countDocuments({})).toBe(0);
});
it('erases only the selected tenant and keeps ordinary deletion nontransactional', async () => {
const conversationId = uuidv4();
await Conversation.create({
conversationId,
user: 'user123',
tenantId: 'tenant-a',
endpoint: EModelEndpoint.agents,
});
await seedResponse(conversationId, 'tenant-a');
await seedResponse(conversationId, 'tenant-b');
const session = jest.spyOn(mongoose, 'startSession');
await tenantStorage.run({ tenantId: 'tenant-a' }, () =>
deleteConvos('user123', { conversationId }),
);
expect(session).not.toHaveBeenCalled();
session.mockRestore();
await runAsSystem(async () => {
expect(
await mongoose.models.StoredResponse.countDocuments({ tenantId: 'tenant-a' }),
).toBe(0);
expect(
await mongoose.models.StoredResponse.countDocuments({ tenantId: 'tenant-b' }),
).toBe(1);
});
});
it('retains payloads when an earlier hook fails but keeps deletion intent closed until retry', async () => {
const conversationId = uuidv4();
await Conversation.create({
conversationId,
user: 'user123',
endpoint: EModelEndpoint.agents,
});
const response = await seedResponse(conversationId);
await expect(
deleteConvos(
'user123',
{ conversationId },
{
beforeDelete: async () => {
throw new Error('drain failed');
},
},
),
).rejects.toThrow('drain failed');
expect(await Conversation.exists({ conversationId })).not.toBeNull();
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(1);
expect(
await createResponseStorage(mongoose).read({ userId: 'user123' }, response.responseId),
).not.toBeNull();
await deleteConvos('user123', { conversationId });
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
});
it('recovers private response cleanup after the root and its messages are already gone', async () => {
const conversationId = uuidv4();
await seedResponse(conversationId);
const result = await deleteConvos('user123', { conversationId });
expect(result.conversationIds).toContain(conversationId);
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
});
it('retries irreversible response erasure when the subsequent parent deletion fails', async () => {
const conversationId = uuidv4();
await Conversation.create({
conversationId,
user: 'user123',
endpoint: EModelEndpoint.agents,
});
await seedResponse(conversationId);
const deletion = jest
.spyOn(Conversation.collection, 'deleteMany')
.mockRejectedValueOnce(new Error('parent delete failed'));
await expect(deleteConvos('user123', { conversationId })).rejects.toThrow(
'parent delete failed',
);
deletion.mockRestore();
expect(await Conversation.exists({ conversationId })).not.toBeNull();
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
await deleteConvos('user123', { conversationId });
expect(await Conversation.exists({ conversationId })).toBeNull();
});
it('allows concurrent erasers to finish without reopening admission', async () => {
const conversationId = uuidv4();
await Conversation.create({
conversationId,
user: 'user123',
endpoint: EModelEndpoint.agents,
});
const response = await seedResponse(conversationId);
await Promise.all([
deleteConvos('user123', { conversationId }, { allowEmpty: true }),
deleteConvos('user123', { conversationId }, { allowEmpty: true }),
]);
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
expect(
await createResponseStorage(mongoose).read({ userId: 'user123' }, response.responseId),
).toBeNull();
});
});
it('retires queued-turn work before each conversation deletion wave', async () => {
const conversationId = uuidv4();
await Conversation.create({

View file

@ -29,6 +29,7 @@ import type {
IMessage,
ISharedLink,
ISubagentThreadReservation,
IUser,
} from '~/types';
import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';
import type { MessageMethods } from './message';
@ -39,6 +40,13 @@ import {
MAX_AGENT_EVENT_ACTOR_SUMMARY_LENGTH,
MAX_AGENT_EVENT_ACTOR_TOOL_NAME_LENGTH,
} from '~/types/convo';
import {
closeResponseScopes,
eraseResponseRecords,
createResponseRetention,
discoverResponseOwners,
createResponseConversationReplacement,
} from './responseScope';
import {
activeExpirationFilter,
buildRetentionVisibilityFilter,
@ -49,6 +57,7 @@ import {
updateChatProjectLastConversationForUser,
} from './chatProject';
import { createChatExpirationDate, createTempChatExpirationDate } from '~/utils/tempChatRetention';
import { getTenantId, SYSTEM_TENANT_ID, runAsSystem } from '~/config/tenantContext';
import { isAgentFadingTier, isAgentFadingTierEntries } from '~/utils/fading';
import { isCompactionSemanticIndexProjection } from '~/types/compaction';
import { withoutMeiliIndexing } from '~/models/plugins/mongoMeili';
@ -668,6 +677,8 @@ export function createConversationMethods(
mongoose: typeof import('mongoose'),
deps?: ConversationMethodDeps,
): ConversationMethods {
const retainResponses = createResponseRetention(mongoose);
const prepareResponseReplacement = createResponseConversationReplacement(mongoose);
let legacyReceiptExpiryCursor: Types.ObjectId | undefined;
/**
@ -2336,12 +2347,13 @@ export function createConversationMethods(
const Conversation = mongoose.models.Conversation as Model<IConversation>;
const stored = await Conversation.findOne({ conversationId, user: userId })
.select({ _id: 1, isTemporary: 1, expiredAt: 1, tags: 1 })
.select({ _id: 1, isTemporary: 1, expiredAt: 1, tags: 1, tenantId: 1 })
.lean<{
_id: Types.ObjectId;
isTemporary?: boolean;
expiredAt?: Date | null;
tags?: string[];
tenantId?: string;
} | null>();
if (!stored) {
return;
@ -2357,6 +2369,7 @@ export function createConversationMethods(
}
}
await retainResponses({ userId, tenantId: stored.tenantId }, conversationId, expiredAt);
let stampedDeadline = stored.expiredAt != null;
if (stored.isTemporary !== true) {
/**
@ -2580,6 +2593,37 @@ export function createConversationMethods(
update.expiredAt = null;
}
const responseDeadline =
update.expiredAt instanceof Date ? update.expiredAt : retentionOnInsert?.expiredAt;
const replacingConversationId = newConversationId && newConversationId !== conversationId;
if (responseDeadline || replacingConversationId) {
const stored = await Conversation.findOne({ user: userId, conversationId })
.select('tenantId expiredAt')
.lean<Pick<IConversation, 'tenantId' | 'expiredAt'>>();
const contextTenant = getTenantId();
const responseOwner = {
userId,
tenantId:
stored?.tenantId ?? (contextTenant === SYSTEM_TENANT_ID ? undefined : contextTenant),
};
// Conversation IDs are shared foreign keys for messages, approvals, jobs
// and checkpoints. newConversationId assigns an initial ID, never renames
// a persisted conversation independently of those resources.
if (
replacingConversationId &&
(stored || !(await prepareResponseReplacement(responseOwner, conversationId)))
) {
return { message: 'Conversation ID cannot be replaced after persistence.' };
}
const deadline = responseDeadline ?? stored?.expiredAt;
if (deadline)
await retainResponses(
responseOwner,
newConversationId || conversationId,
stored?.expiredAt && stored.expiredAt < deadline ? stored.expiredAt : deadline,
);
}
const createdAtOnInsert =
metadata?.createdAtOnInsert instanceof Date &&
!Number.isNaN(metadata.createdAtOnInsert.getTime())
@ -2652,6 +2696,9 @@ export function createConversationMethods(
const baseFilter = {
conversationId,
user: userId,
// Initial ID assignment must never match a source inserted after the
// absence check. Upsert may create the new ID, but cannot rename that row.
...(replacingConversationId ? { _id: { $exists: false } } : {}),
...(metadata?.titleSource === 'generated' && {
titleSetByUser: { $ne: true },
title: { $in: [null, '', 'New Chat'] },
@ -2773,6 +2820,11 @@ export function createConversationMethods(
interfaceConfig,
conversation.isTemporary === true,
);
await retainResponses(
{ userId, tenantId: conversation.tenantId },
conversationId,
deadline,
);
const result = await Conversation.updateOne(
{
_id: conversation._id,
@ -4046,12 +4098,40 @@ export function createConversationMethods(
.select('conversationId tenantId chatProjectId tags')
.lean<DeletionConversation[]>();
const recoveryConversationIds: string[] = [];
const recoveryResponseScopes: DeletionConversation[] = [];
const responseGroups = (wave: DeletionConversation[]) => {
const groups = new Map<string | undefined, string[]>();
for (const conversation of wave) {
const tenantId = conversation.tenantId || undefined;
const ids = groups.get(tenantId) ?? [];
ids.push(conversation.conversationId);
groups.set(tenantId, ids);
}
return groups;
};
const closeResponses = async (wave: DeletionConversation[]) => {
for (const [tenantId, ids] of responseGroups(wave))
await closeResponseScopes(mongoose, { userId: user, tenantId }, ids);
};
const eraseResponses = async (wave: DeletionConversation[]) => {
for (const [tenantId, ids] of responseGroups(wave))
await eraseResponseRecords(mongoose, { userId: user, tenantId }, ids);
};
if (!conversations.length && typeof filter.conversationId === 'string') {
// Authorized delayed reservations retain the globally unique user's actual tenant.
// Fence it before probing legacy payloads, including when no response exists yet.
const owner = isValidObjectIdString(user)
? await (mongoose.models.User as Model<IUser>).findById(user).select('tenantId').lean()
: null;
if (owner)
await closeResponses([
{ conversationId: filter.conversationId, tenantId: owner.tenantId },
]);
/** A prior attempt may have deleted the root before a descendant read failed.
* Resume from immutable root lineage and retain the root id for message,
* checkpoint, and tool cleanup. The message probe distinguishes that partial
* commit from a conversation id that never existed. */
const [descendants, rootMessages] = await Promise.all([
const [descendants, rootMessages, responseOwners] = await Promise.all([
retryCascadeOperation(() =>
Conversation.find({
user,
@ -4061,8 +4141,20 @@ export function createConversationMethods(
.lean<DeletionConversation[]>(),
),
getMessages({ user, conversationId: filter.conversationId }, '_id', { limit: 1 }),
discoverResponseOwners(mongoose, user, filter.conversationId),
]);
if (descendants.length === 0 && rootMessages.length === 0 && options?.allowEmpty !== true) {
for (const responseOwner of responseOwners) {
recoveryResponseScopes.push({
conversationId: filter.conversationId,
tenantId: responseOwner.tenantId,
});
}
if (
descendants.length === 0 &&
rootMessages.length === 0 &&
recoveryResponseScopes.length === 0 &&
options?.allowEmpty !== true
) {
throw new Error('Conversation not found or already deleted.');
}
conversations = descendants;
@ -4132,6 +4224,7 @@ export function createConversationMethods(
break;
}
const waveIds = wave.map((conversation) => conversation.conversationId);
await closeResponses(wave);
await deps?.deleteAgentQueuedTurns?.(
user,
wave.map((conversation) => ({
@ -4147,6 +4240,8 @@ export function createConversationMethods(
conversationId: { $in: waveIds },
}),
);
// Like subagent content, response erasure is an irreversible part of deletion intent.
await eraseResponses(wave);
const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } });
if (result.deletedCount > 0) {
/** Result erasure is irreversible. Keep receipts intact when a
@ -4181,6 +4276,12 @@ export function createConversationMethods(
];
if (recoveryConversationIds.length > 0) {
// Each command retains an explicit owner/partition predicate. Request tenant
// middleware must not replace the partitions discovered by recovery.
await runAsSystem(async () => {
await closeResponses(recoveryResponseScopes);
await eraseResponses(recoveryResponseScopes);
});
await deps?.deleteAgentQueuedTurns?.(
user,
recoveryConversationIds.map((conversationId) => ({

View file

@ -69,6 +69,20 @@ describe('conversation import cleanup methods', () => {
text: record.conversationId,
})),
);
await mongoose.models.StoredResponse.insertMany(
records.map((record, index) => ({
userId: record.user,
tenantId: record.tenantId,
conversationId: record.conversationId,
responseId: `response-${index}`,
requestKey: `request-${index}`,
fingerprint: 'request',
agentId: 'agent',
requestJSON: '{"input":"private"}',
parentMessageId: 'root',
status: 'queued',
})),
);
await mongoose.models.ChatProject.updateOne(
{ _id: project._id },
{
@ -99,6 +113,9 @@ describe('conversation import cleanup methods', () => {
});
expect(remaining.conversations).toHaveLength(3);
expect(remaining.messages).toHaveLength(3);
expect(await runAsSystem(async () => mongoose.models.StoredResponse.countDocuments({}))).toBe(
3,
);
expect(
remaining.conversations.some(
(record) =>
@ -158,10 +175,17 @@ describe('conversation import cleanup methods', () => {
const deleteMessages = jest.fn().mockResolvedValue(undefined);
const findProjects = jest.fn().mockResolvedValue([]);
const deleteConversations = jest.fn().mockResolvedValue(undefined);
const responseScopes = { bulkWrite: jest.fn().mockResolvedValue(undefined) };
const cleanupMongoose = {
model: jest.fn().mockReturnValue(responseScopes),
models: {
Message: { deleteMany: deleteMessages },
Conversation: { distinct: findProjects, deleteMany: deleteConversations },
ResponseScope: responseScopes,
StoredResponse: {
updateMany: jest.fn().mockResolvedValue(undefined),
deleteMany: jest.fn().mockResolvedValue(undefined),
},
},
} as unknown as typeof mongoose;
const methods = createConversationImportMethods(cleanupMongoose);

View file

@ -1,5 +1,6 @@
import type { FilterQuery, Model } from 'mongoose';
import type { IConversation, IMessage } from '~/types';
import { closeResponseScopes, eraseResponseRecords } from './responseScope';
import { runAsSystem, tenantStorage } from '~/config/tenantContext';
import { refreshChatProjectStatsForUser } from './chatProject';
@ -77,6 +78,9 @@ export function createConversationImportMethods(
affectedProjectIds.add(String(projectId));
}
}
const owner = { userId: scope.user, tenantId: scope.tenantId };
await closeResponseScopes(mongoose, owner, conversationIds);
await eraseResponseRecords(mongoose, owner, conversationIds);
await Conversation.deleteMany(filter);
await mongoose.models.SubagentContent?.deleteMany(filter);
}

View file

@ -67,6 +67,7 @@ import {
import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCategory';
/* Agent API Keys */
import { createAgentApiKeyMethods, type AgentApiKeyMethods } from './agentApiKey';
/* MCP Servers */
import { createMCPServerMethods, type MCPServerMethods } from './mcpServer';
import { createCodeEnvironmentMethods, type CodeEnvironmentMethods } from './codeEnvironment';

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,691 @@
import type {
ResponseOwner,
StoredResponseRecord,
StoredResponseStatus,
ResponseReviewCheckpoint,
ResponseResumeIntent,
ResponseTerminalReceipt,
} from '~/models/response';
import {
responseScopeKeys,
responseOwnerFilter,
isResponseScopeClosed,
initializeResponseScopes,
} from './responseScope';
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { createResponseScopeModel } from '~/models/responseScope';
import { createStoredResponseModel } from '~/models/response';
import { getTransactionSupport } from '~/utils/transactions';
import { activeExpirationFilter } from '~/utils/retention';
import { createIndexesWithRetry } from '~/utils/retry';
export type {
ResponseOwner,
StoredResponseRecord,
StoredResponseStatus,
ResponseReviewCheckpoint,
ResponseResumeIntent,
ResponseTerminalReceipt,
} from '~/models/response';
export interface ResponseReservation extends ResponseOwner {
expiredAt?: Date;
isTemporary?: boolean;
responseId: string;
requestKey: string;
fingerprint: string;
agentId: string;
sourceKeyId?: string;
requestJSON: string;
conversationId: string;
parentMessageId: string;
}
export type ResponseReservationResult =
| { ok: true; record: StoredResponseRecord; replayed: boolean }
| { ok: false; code: 'idempotency_conflict' | 'response_unavailable' };
export type ResponseMutationResult =
| { ok: true; record: StoredResponseRecord }
| { ok: false; code: 'response_conflict' };
/** Internal drain identity only; expired requests, answers and decisions are never returned. */
export type ResponseCleanupIdentity = Pick<
StoredResponseRecord,
| 'userId'
| 'tenantId'
| 'responseId'
| 'agentId'
| 'conversationId'
| 'generationCreatedAt'
| 'expiredAt'
> & {
/** Cleanup authority only; never permission to read output or execute a provider. */
retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired';
checkpointNamespace?: string;
checkpointConfigJSON?: string;
requiresCheckpointCleanup?: boolean;
};
export interface ResponseResumeReservation
extends Omit<ResponseResumeIntent, 'state' | 'reservedAt' | 'admittedAt'> {
generationCreatedAt: number;
}
/** Replays are receipts, not permission to enqueue or execute terminal/cancelled runs. */
export type ResponseResumeResult =
| { ok: true; record: StoredResponseRecord; replayed: boolean }
| { ok: false; code: 'resume_conflict' };
export interface ResponseStorage {
stageTerminal(
owner: ResponseOwner,
responseId: string,
revision: number,
receipt: ResponseTerminalReceipt,
): Promise<ResponseMutationResult>;
finishTerminal(
owner: ResponseOwner,
responseId: string,
revision: number,
receipt: ResponseTerminalReceipt,
): Promise<ResponseMutationResult>;
publishReview(
owner: ResponseOwner,
responseId: string,
revision: number,
generationCreatedAt: number,
review: ResponseReviewCheckpoint,
responseJSON: string,
): Promise<ResponseMutationResult>;
reserveResume(
owner: ResponseOwner,
responseId: string,
input: ResponseResumeReservation,
now: number,
): Promise<ResponseResumeResult>;
admitResume(
owner: ResponseOwner,
responseId: string,
intentId: string,
generationCreatedAt: number,
providerExecutionId: string,
responseJSON: string,
now: number,
): Promise<ResponseMutationResult>;
reserve(input: ResponseReservation): Promise<ResponseReservationResult>;
read(owner: ResponseOwner, responseId: string): Promise<StoredResponseRecord | null>;
readCleanupIdentity(
owner: ResponseOwner,
responseId: string,
retained?: ResponseCleanupIdentity,
): Promise<ResponseCleanupIdentity | null>;
attachDelivery(
owner: ResponseOwner,
responseId: string,
deliveryKey: string,
): Promise<ResponseMutationResult>;
publish(
owner: ResponseOwner,
responseId: string,
revision: number,
update: { status: StoredResponseStatus; responseJSON: string; generationCreatedAt?: number },
): Promise<ResponseMutationResult>;
requestCancellation(
owner: ResponseOwner,
responseId: string,
now: number,
): Promise<StoredResponseRecord | null>;
}
const active = ['queued', 'in_progress', 'requires_action'];
function sameTerminal(a: ResponseTerminalReceipt, b: ResponseTerminalReceipt): boolean {
return (
a.generationCreatedAt === b.generationCreatedAt &&
a.providerExecutionId === b.providerExecutionId &&
a.ownerId === b.ownerId &&
a.status === b.status &&
a.responseJSON === b.responseJSON &&
a.checkpointNamespace === b.checkpointNamespace &&
a.checkpointConfigJSON === b.checkpointConfigJSON
);
}
/** Authorization precedes this storage boundary; every read/write remains owner- and tenant-scoped. */
export function createResponseStorage(mongoose: typeof import('mongoose')): ResponseStorage {
const model = createStoredResponseModel(mongoose);
let indexPromise: Promise<void> | undefined;
let transactionSupport: boolean | null = null;
function ensureIndexes() {
indexPromise ??= createIndexesWithRetry(model)
.then(() => undefined)
.catch((error: unknown) => {
indexPromise = undefined;
throw error;
});
return indexPromise;
}
function ownerScope(owner: ResponseOwner) {
if (!owner.userId || owner.tenantId === '__SYSTEM__')
throw new TypeError('Invalid response owner');
const tenantId = getTenantId() || undefined;
if (
tenantId != null &&
tenantId !== SYSTEM_TENANT_ID &&
tenantId !== (owner.tenantId || undefined)
)
throw new TypeError('Response tenant does not match authenticated scope');
return responseOwnerFilter(owner);
}
const scope = (owner: ResponseOwner) => ({
...ownerScope(owner),
$and: [activeExpirationFilter<StoredResponseRecord>()],
});
return {
async stageTerminal(owner, responseId, revision, receipt) {
const pendingTerminal: ResponseTerminalReceipt = {
generationCreatedAt: receipt.generationCreatedAt,
providerExecutionId: receipt.providerExecutionId,
...(receipt.ownerId == null ? {} : { ownerId: receipt.ownerId }),
status: receipt.status,
responseJSON: receipt.responseJSON,
...(receipt.checkpointNamespace == null
? {}
: { checkpointNamespace: receipt.checkpointNamespace }),
...(receipt.checkpointConfigJSON == null
? {}
: { checkpointConfigJSON: receipt.checkpointConfigJSON }),
};
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
revision,
generationCreatedAt: receipt.generationCreatedAt,
status: { $in: active },
pendingTerminal: { $exists: false },
$or: [
{
'resume.state': 'admitted',
'resume.providerExecutionId': receipt.providerExecutionId,
},
{
'resume.state': { $ne: 'admitted' },
'review.providerExecutionId': receipt.providerExecutionId,
},
{ review: { $exists: false }, resume: { $exists: false } },
],
},
{ $set: { pendingTerminal }, $inc: { revision: 1 } },
{ new: true, runValidators: true },
)
.lean();
if (record) return { ok: true, record };
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
if (
existing?.generationCreatedAt === receipt.generationCreatedAt &&
existing.pendingTerminal &&
sameTerminal(existing.pendingTerminal, pendingTerminal)
)
return { ok: true, record: existing };
return { ok: false, code: 'response_conflict' };
},
async finishTerminal(owner, responseId, revision, receipt) {
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
revision,
generationCreatedAt: receipt.generationCreatedAt,
status: { $in: active },
'pendingTerminal.generationCreatedAt': receipt.generationCreatedAt,
'pendingTerminal.providerExecutionId': receipt.providerExecutionId,
'pendingTerminal.ownerId': receipt.ownerId ?? { $exists: false },
'pendingTerminal.status': receipt.status,
'pendingTerminal.responseJSON': receipt.responseJSON,
'pendingTerminal.checkpointNamespace': receipt.checkpointNamespace ?? {
$exists: false,
},
'pendingTerminal.checkpointConfigJSON': receipt.checkpointConfigJSON ?? {
$exists: false,
},
},
{
$set: { status: receipt.status, responseJSON: receipt.responseJSON },
$unset: { pendingTerminal: 1 },
$inc: { revision: 1 },
},
{ new: true },
)
.lean();
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
},
async publishReview(owner, responseId, revision, generationCreatedAt, review, responseJSON) {
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
revision,
generationCreatedAt,
status: { $in: ['in_progress', 'requires_action'] },
pendingTerminal: { $exists: false },
cancellationRequestedAt: { $exists: false },
$or: [
{ review: { $exists: false }, resume: { $exists: false } },
{
'review.actionId': review.actionId,
'review.providerExecutionId': review.providerExecutionId,
resume: { $exists: false },
},
{
'resume.state': 'admitted',
'resume.providerExecutionId': review.providerExecutionId,
},
],
},
{
$set: {
status: 'requires_action',
review: {
actionId: review.actionId,
providerExecutionId: review.providerExecutionId,
expiresAt: review.expiresAt,
partialResponseJSON: review.partialResponseJSON,
...(review.visibleOutputTokens == null
? {}
: { visibleOutputTokens: review.visibleOutputTokens }),
...(review.pendingActionJSON == null
? {}
: { pendingActionJSON: review.pendingActionJSON }),
...(review.checkpointConfigJSON == null
? {}
: { checkpointConfigJSON: review.checkpointConfigJSON }),
...(review.checkpointId == null ? {} : { checkpointId: review.checkpointId }),
...(review.checkpointNs == null ? {} : { checkpointNs: review.checkpointNs }),
},
responseJSON,
},
$unset: { resume: 1 },
$inc: { revision: 1 },
},
{ new: true, runValidators: true },
)
.lean();
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
},
async reserveResume(owner, responseId, input, now) {
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume reservation time');
if (input.providerExecutionId === input.previousProviderExecutionId)
throw new TypeError('Resume requires a new provider execution identity');
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
generationCreatedAt: input.generationCreatedAt,
status: 'requires_action',
pendingTerminal: { $exists: false },
cancellationRequestedAt: { $exists: false },
resume: { $exists: false },
'review.actionId': input.actionId,
'review.providerExecutionId': input.previousProviderExecutionId,
'review.expiresAt': { $gt: now },
},
{
$set: {
resume: {
intentId: input.intentId,
actionId: input.actionId,
previousProviderExecutionId: input.previousProviderExecutionId,
providerExecutionId: input.providerExecutionId,
fingerprint: input.fingerprint,
resolutionJSON: input.resolutionJSON,
...(input.decisionJSON == null ? {} : { decisionJSON: input.decisionJSON }),
state: 'reserved',
reservedAt: now,
},
},
$inc: { revision: 1 },
},
{ new: true, runValidators: true },
)
.lean();
if (record) return { ok: true, record, replayed: false };
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
const intent = existing?.resume;
if (
existing?.generationCreatedAt === input.generationCreatedAt &&
intent &&
intent.actionId === input.actionId &&
intent.previousProviderExecutionId === input.previousProviderExecutionId &&
intent.fingerprint === input.fingerprint &&
intent.resolutionJSON === input.resolutionJSON &&
intent.decisionJSON === input.decisionJSON
)
return { ok: true, record: existing, replayed: true };
return { ok: false, code: 'resume_conflict' };
},
async admitResume(
owner,
responseId,
intentId,
generationCreatedAt,
providerExecutionId,
responseJSON,
now,
) {
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume admission time');
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
generationCreatedAt,
status: 'requires_action',
pendingTerminal: { $exists: false },
cancellationRequestedAt: { $exists: false },
'resume.intentId': intentId,
'resume.providerExecutionId': providerExecutionId,
'resume.state': 'reserved',
},
{
$set: {
status: 'in_progress',
responseJSON,
'resume.state': 'admitted',
'resume.admittedAt': now,
},
$inc: { revision: 1 },
},
{ new: true },
)
.lean();
if (record) return { ok: true, record };
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
if (
existing?.generationCreatedAt === generationCreatedAt &&
existing.resume?.intentId === intentId &&
existing.resume.providerExecutionId === providerExecutionId &&
existing.resume.state === 'admitted'
)
return { ok: true, record: existing };
return { ok: false, code: 'response_conflict' };
},
async reserve(input) {
if (
input.expiredAt != null &&
(!(input.expiredAt instanceof Date) || !Number.isFinite(input.expiredAt.getTime()))
)
throw new TypeError('Invalid response expiration');
const filter = { ...scope(input), requestKey: input.requestKey };
transactionSupport = await getTransactionSupport(mongoose, transactionSupport);
if (!transactionSupport) return { ok: false, code: 'response_unavailable' };
await ensureIndexes();
const prior = await model.findOne(filter).select('conversationId').lean();
const keys = responseScopeKeys(input, prior?.conversationId ?? input.conversationId);
await initializeResponseScopes(mongoose, keys);
const session = await mongoose.startSession();
try {
let result: ResponseReservationResult | undefined;
await session.withTransaction(async () => {
// A write, not a snapshot read: erasure waits for admitted insertions to commit.
const scopes = createResponseScopeModel(mongoose);
const touched = await scopes.updateOne(
{ _id: keys[0], closed: false },
{ $inc: { revision: 1 } },
{ session },
);
if (touched.matchedCount !== 1) {
result = { ok: false, code: 'response_unavailable' };
return;
}
const existing = await model.findOne(filter).session(session).lean();
const conversationKey = responseScopeKeys(
input,
existing?.conversationId ?? input.conversationId,
)[1];
const conversation = await scopes
.findOneAndUpdate(
{ _id: conversationKey, closed: false },
{
$inc: { revision: 1 },
...(input.expiredAt ? { $min: { expiredAt: input.expiredAt } } : {}),
},
{ session, new: true },
)
.lean();
if (!conversation) {
result = { ok: false, code: 'response_unavailable' };
return;
}
if (conversation.expiredAt) {
await model.updateMany(
{
...responseOwnerFilter(input),
conversationId: existing?.conversationId ?? input.conversationId,
$or: [{ expiredAt: null }, { expiredAt: { $gt: conversation.expiredAt } }],
},
{ $set: { expiredAt: conversation.expiredAt } },
{ session },
);
if (existing && (!existing.expiredAt || conversation.expiredAt < existing.expiredAt))
existing.expiredAt = conversation.expiredAt;
if (conversation.expiredAt.getTime() <= Date.now()) {
result = { ok: false, code: 'response_unavailable' };
return;
}
}
if (existing) {
result =
existing.fingerprint === input.fingerprint
? { ok: true, record: existing, replayed: true }
: { ok: false, code: 'idempotency_conflict' };
return;
}
const [created] = await model.create(
[
{
...input,
...(conversation.expiredAt ? { expiredAt: conversation.expiredAt } : {}),
status: 'queued',
revision: 0,
},
],
{
session,
},
);
result = { ok: true, record: created.toObject(), replayed: false };
});
if (!result) throw new Error('Response reservation did not commit');
return result;
} catch (error) {
if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 11000)
throw error;
const existing = await model.findOne(filter).lean();
if (!existing) {
if (await model.exists({ ...responseOwnerFilter(input), requestKey: input.requestKey }))
return { ok: false, code: 'response_unavailable' };
throw error;
}
if (await isResponseScopeClosed(mongoose, input, existing.conversationId))
return { ok: false, code: 'response_unavailable' };
return existing.fingerprint === input.fingerprint
? { ok: true, record: existing, replayed: true }
: { ok: false, code: 'idempotency_conflict' };
} finally {
await session.endSession();
}
},
async read(owner, responseId) {
return model.findOne({ ...scope(owner), responseId }).lean();
},
async readCleanupIdentity(owner, responseId, retained) {
const record = await model
.findOne({ ...ownerScope(owner), responseId })
.select(
'userId tenantId responseId agentId conversationId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId',
)
.lean();
// Retained data comes only from an internal job/receipt, never request input.
// Validate it even when the row exists before using any fallback checkpoint fields.
const matchingRetained =
retained &&
retained.responseId === responseId &&
retained.userId === owner.userId &&
(retained.tenantId || undefined) === (owner.tenantId || undefined) &&
(!record ||
(retained.conversationId === record.conversationId &&
retained.agentId === record.agentId &&
retained.generationCreatedAt === record.generationCreatedAt))
? retained
: undefined;
const source = record ?? matchingRetained;
if (!source) return null;
const now = new Date();
const [ownerKey, conversationKey] = responseScopeKeys(owner, source.conversationId);
const anchors = await createResponseScopeModel(mongoose)
.find({ _id: { $in: [ownerKey, conversationKey] } })
.select('closed expiredAt')
.lean();
const ownerAnchor = anchors.find((anchor) => anchor._id === ownerKey);
const conversationAnchor = anchors.find((anchor) => anchor._id === conversationKey);
const expiredAt =
conversationAnchor?.expiredAt &&
(!source.expiredAt || conversationAnchor.expiredAt < source.expiredAt)
? conversationAnchor.expiredAt
: source.expiredAt;
let retirement: ResponseCleanupIdentity['retirement'];
if (ownerAnchor?.closed) retirement = 'owner_deleted';
else if (conversationAnchor?.closed) retirement = 'conversation_deleted';
else if (expiredAt && expiredAt <= now) retirement = 'expired';
// A retained deadline alone cannot prove why an authoritative row disappeared.
if (
!record &&
!ownerAnchor?.closed &&
!conversationAnchor?.closed &&
!(conversationAnchor?.expiredAt && conversationAnchor.expiredAt <= now)
)
return null;
// A staged checkpoint is one binding: never fill an incomplete terminal
// binding with settings from a prior review or retained provider segment.
const terminalCheckpoint = record?.pendingTerminal;
const hasTerminalCheckpoint =
terminalCheckpoint?.checkpointNamespace != null ||
terminalCheckpoint?.checkpointConfigJSON != null;
const checkpointNamespace = hasTerminalCheckpoint
? terminalCheckpoint?.checkpointNamespace
: matchingRetained?.checkpointNamespace;
const checkpointConfigJSON = hasTerminalCheckpoint
? terminalCheckpoint?.checkpointConfigJSON
: (record?.review?.checkpointConfigJSON ?? matchingRetained?.checkpointConfigJSON);
const {
userId,
tenantId,
responseId: id,
agentId,
conversationId,
generationCreatedAt,
} = source;
return {
userId,
tenantId,
responseId: id,
agentId,
conversationId,
generationCreatedAt,
expiredAt,
retirement,
checkpointNamespace,
checkpointConfigJSON,
requiresCheckpointCleanup: Boolean(
record?.review ||
checkpointNamespace ||
checkpointConfigJSON ||
matchingRetained?.requiresCheckpointCleanup,
),
};
},
async attachDelivery(owner, responseId, deliveryKey) {
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
status: { $in: active },
$or: [{ deliveryKey: { $exists: false } }, { deliveryKey }],
},
{ $set: { deliveryKey } },
{ new: true },
)
.lean();
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
},
async publish(owner, responseId, revision, update) {
if (
update.generationCreatedAt != null &&
(!Number.isSafeInteger(update.generationCreatedAt) || update.generationCreatedAt <= 0)
)
throw new TypeError('Invalid response generation identity');
let predecessors = active;
if (update.status === 'queued') predecessors = ['queued'];
if (update.status === 'in_progress') predecessors = ['queued', 'in_progress'];
const record = await model
.findOneAndUpdate(
{
...scope(owner),
responseId,
revision,
status: { $in: predecessors },
pendingTerminal: { $exists: false },
...(update.status === 'in_progress'
? {
$and: [
activeExpirationFilter<StoredResponseRecord>(),
{
$or: [
{ status: 'in_progress' },
{ cancellationRequestedAt: { $exists: false } },
],
},
...(update.generationCreatedAt == null
? [{ generationCreatedAt: { $gt: 0 } }]
: []),
],
}
: {}),
...(update.generationCreatedAt == null
? {}
: {
$or: [
{ generationCreatedAt: { $exists: false } },
{ generationCreatedAt: update.generationCreatedAt },
],
}),
},
{ $set: update, $inc: { revision: 1 } },
{ new: true },
)
.lean();
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
},
async requestCancellation(owner, responseId, now) {
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid cancellation time');
await model.updateOne(
{
...scope(owner),
responseId,
status: { $in: active },
cancellationRequestedAt: { $exists: false },
pendingTerminal: { $exists: false },
},
{ $set: { cancellationRequestedAt: now } },
);
return model.findOne({ ...scope(owner), responseId }).lean();
},
};
}

View file

@ -0,0 +1,224 @@
import { createHash } from 'node:crypto';
import type { ResponseOwner } from '~/models/response';
import { createResponseScopeModel } from '~/models/responseScope';
import { createStoredResponseModel } from '~/models/response';
import { getTransactionSupport } from '~/utils/transactions';
import { createIndexesWithRetry } from '~/utils/retry';
import { runAsSystem } from '~/config/tenantContext';
/** Response-bound conversation IDs are immutable: jobs and checkpoints retain that identity.
* An empty source scope is closed before replacement so delayed admission cannot orphan data. */
export function createResponseConversationReplacement(
mongoose: typeof import('mongoose'),
): (owner: ResponseOwner, conversationId: string) => Promise<boolean> {
let supported: boolean | null = null;
return async (owner, conversationId) => {
supported = await getTransactionSupport(mongoose, supported);
const records = createStoredResponseModel(mongoose);
const filter = { ...responseOwnerFilter(owner), conversationId };
if (!supported) {
// Reservation fails closed on standalone deployments, so no admission can race this check.
if (await records.exists(filter)) return false;
await closeResponseScopes(mongoose, owner, [conversationId]);
return true;
}
const key = responseScopeKeys(owner, conversationId)[1];
await initializeResponseScopes(mongoose, [key]);
const session = await mongoose.startSession();
try {
let allowed = false;
await session.withTransaction(async () => {
const scopes = createResponseScopeModel(mongoose);
// Serialize with the reservation's write to this same anchor before checking rows.
await scopes.updateOne({ _id: key }, { $inc: { revision: 1 } }, { session });
if (await records.exists(filter).session(session)) {
allowed = false;
return;
}
await scopes.updateOne({ _id: key }, { $set: { closed: true } }, { session });
allowed = true;
});
return allowed;
} finally {
await session.endSession();
}
};
}
/** Authorized missing-root recovery, restricted to one globally unique owner/conversation. */
export async function discoverResponseOwners(
mongoose: typeof import('mongoose'),
userId: string,
conversationId: string,
): Promise<ResponseOwner[]> {
return runAsSystem(async () => {
const model = createStoredResponseModel(mongoose);
const filter = { userId, conversationId };
const [tenants, tenantless] = await Promise.all([
model.distinct('tenantId', filter),
model.exists({ ...filter, tenantId: { $in: [null, ''] } }),
]);
const owners: ResponseOwner[] = tenants
.filter((tenantId): tenantId is string => typeof tenantId === 'string' && tenantId !== '')
.map((tenantId) => ({ userId, tenantId }));
if (tenantless) owners.push({ userId });
return owners;
});
}
/** Retention intent is monotonic, like erasure: failed parent saves never reopen private data. */
export function createResponseRetention(
mongoose: typeof import('mongoose'),
): (owner: ResponseOwner, conversationId: string, expiredAt: Date) => Promise<void> {
let supported: boolean | null = null;
let indexes: Promise<void> | undefined;
return async (owner, conversationId, expiredAt) => {
if (!(expiredAt instanceof Date) || !Number.isFinite(expiredAt.getTime()))
throw new TypeError('Invalid response expiration');
const key = responseScopeKeys(owner, conversationId)[1];
await initializeResponseScopes(mongoose, [key]);
const scopes = createResponseScopeModel(mongoose);
const records = createStoredResponseModel(mongoose);
supported = await getTransactionSupport(mongoose, supported);
if (supported) {
indexes ??= createIndexesWithRetry(records)
.then(() => undefined)
.catch((error: unknown) => {
indexes = undefined;
throw error;
});
await indexes;
}
const apply = async (session?: import('mongoose').ClientSession) => {
const anchor = await scopes
.findOneAndUpdate(
{ _id: key },
{ $min: { expiredAt }, $inc: { revision: 1 } },
{ new: true, ...(session ? { session } : {}) },
)
.lean();
if (!anchor?.expiredAt) throw new Error('Response retention scope unavailable');
await records.updateMany(
{
...responseOwnerFilter(owner),
conversationId,
$or: [{ expiredAt: null }, { expiredAt: { $gt: anchor.expiredAt } }],
},
{ $set: { expiredAt: anchor.expiredAt } },
session ? { session } : {},
);
};
if (!supported) return apply();
const session = await mongoose.startSession();
try {
await session.withTransaction(() => apply(session));
} finally {
await session.endSession();
}
};
}
/** The tenant policy treats missing, null and empty tenant IDs as one partition. */
export function responseOwnerFilter(owner: ResponseOwner): {
userId: string;
tenantId: string | { $in: Array<null | string> };
} {
return { userId: owner.userId, tenantId: owner.tenantId || { $in: [null, ''] } };
}
export async function eraseResponseRecords(
mongoose: typeof import('mongoose'),
owner: ResponseOwner,
conversationIds?: readonly string[],
): Promise<void> {
await createStoredResponseModel(mongoose).deleteMany({
...responseOwnerFilter(owner),
...(conversationIds ? { conversationId: { $in: conversationIds } } : {}),
});
}
function scopeKey(owner: ResponseOwner, conversationId?: string): string {
return createHash('sha256')
.update(JSON.stringify([owner.tenantId || null, owner.userId, conversationId ?? null]))
.digest('hex');
}
export function responseScopeKeys(owner: ResponseOwner, conversationId: string): string[] {
return [scopeKey(owner), scopeKey(owner, conversationId)];
}
/** Initializes only metadata; a late initializer can never reopen an erased scope. */
export async function initializeResponseScopes(
mongoose: typeof import('mongoose'),
keys: string[],
): Promise<void> {
const model = createResponseScopeModel(mongoose);
for (const key of keys) {
try {
await model.updateOne(
{ _id: key },
{ $setOnInsert: { closed: false, revision: 0 } },
{ upsert: true },
);
} catch (error) {
if (
!(error instanceof mongoose.mongo.MongoServerError) ||
error.code !== 11000 ||
!(await model.exists({ _id: key }))
)
throw error;
}
}
}
export async function isResponseScopeClosed(
mongoose: typeof import('mongoose'),
owner: ResponseOwner,
conversationId: string,
): Promise<boolean> {
return (
(await createResponseScopeModel(mongoose).exists({
_id: { $in: responseScopeKeys(owner, conversationId) },
closed: true,
})) != null
);
}
/** Irreversible deletion intent. Ordinary retries complete erasure without readmitting work. */
export async function closeResponseScopes(
mongoose: typeof import('mongoose'),
owner: ResponseOwner,
conversationIds?: string[],
): Promise<void> {
const model = createResponseScopeModel(mongoose);
const keys = conversationIds?.map((id) => scopeKey(owner, id)) ?? [scopeKey(owner)];
// Bound each command independently of the number of conversations being removed.
for (let offset = 0; offset < keys.length; offset += 250) {
const operations = keys.slice(offset, offset + 250).map((key) => ({
updateOne: {
filter: { _id: key },
update: { $set: { closed: true }, $setOnInsert: { revision: 0 } },
upsert: true,
},
}));
// Keys already hash tenant + owner; these payload-free anchors have no tenantId column.
// eslint-disable-next-line no-restricted-syntax
const close = () => model.bulkWrite(operations, { ordered: false });
try {
await close();
} catch (error) {
if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 11000) throw error;
await close();
}
}
await createStoredResponseModel(mongoose).updateMany(
{
...responseOwnerFilter(owner),
...(conversationIds ? { conversationId: { $in: conversationIds } } : {}),
status: { $in: ['queued', 'in_progress', 'requires_action'] },
cancellationRequestedAt: { $exists: false },
pendingTerminal: { $exists: false },
},
{ $set: { cancellationRequestedAt: Date.now() } },
);
}

View file

@ -0,0 +1,30 @@
import { waitForPause } from './test-helpers';
it('reaches a pause without waiting for the blocked operation', async () => {
await expect(
waitForPause(Promise.resolve(), new Promise<void>(() => undefined)),
).resolves.toBeUndefined();
});
it('rejects an operation that completes without reaching its pause', async () => {
await expect(
waitForPause(new Promise<void>(() => undefined), Promise.resolve('finished')),
).rejects.toThrow('Operation completed before reaching its pause');
});
it('preserves an operation failure before its pause', async () => {
const failure = new Error('write failed');
await expect(
waitForPause(new Promise<void>(() => undefined), Promise.reject(failure)),
).rejects.toBe(failure);
});
it('observes a late operation failure after its pause was reached', async () => {
let fail!: (reason: Error) => void;
const operation = new Promise<void>((_resolve, reject) => {
fail = reject;
});
await expect(waitForPause(Promise.resolve(), operation)).resolves.toBeUndefined();
fail(new Error('late write failure'));
await new Promise<void>((resolve) => setImmediate(resolve));
});

View file

@ -1,5 +1,15 @@
import { gptPointReleaseFamily } from 'librechat-data-provider';
/** Fail promptly if an operation settles without reaching the expected pause. */
export function waitForPause<T>(pause: Promise<void>, operation: Promise<T>): Promise<void> {
return Promise.race([
pause,
operation.then(() => {
throw new Error('Operation completed before reaching its pause');
}),
]);
}
/**
* Inlined utility functions previously imported from @librechat/api.
* These are used only by test files in data-schemas.

View file

@ -8,6 +8,9 @@ import {
import type * as t from '~/types';
import { createToolApprovalGrantModel } from '~/models/toolApprovalGrant';
import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';
import { createStoredResponseModel } from '~/models/response';
import { runAsSystem } from '~/config/tenantContext';
import { createResponseStorage } from './response';
import balanceSchema from '~/schema/balance';
import userSchema from '~/schema/user';
@ -2362,3 +2365,85 @@ describe('personal grant cleanup before account deletion', () => {
}
});
});
describe('private response cleanup before account deletion', () => {
async function createResponses() {
const user = await User.create({ email: 'delete-responses@example.com', provider: 'local' });
const userId = user._id.toString();
const model = createStoredResponseModel(mongoose);
await runAsSystem(() =>
model.create(
[{ userId }, { userId, tenantId: 'another-tenant' }, { userId: 'another-owner' }].map(
(scope, index) => ({
...scope,
responseId: `response-${index}`,
requestKey: `request-${index}`,
fingerprint: 'request',
agentId: 'agent',
requestJSON: '{"input":"private request"}',
responseJSON: '{"output":"private reply"}',
conversationId: 'conversation',
parentMessageId: 'root',
status: 'completed',
}),
),
),
);
return { userId, model };
}
it('removes private payloads before committing deletion and preserves other owners and tenants', async () => {
const { userId, model } = await createResponses();
const cleanup = jest.spyOn(model, 'deleteMany');
const deletion = jest.spyOn(User, 'deleteOne');
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
expect(cleanup).toHaveBeenCalledWith({ userId, tenantId: { $in: [null, ''] } });
expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(deletion.mock.invocationCallOrder[0]);
await runAsSystem(async () => {
expect(await model.countDocuments({ userId, tenantId: null })).toBe(0);
expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(1);
expect(await model.countDocuments({ userId: 'another-owner' })).toBe(1);
});
});
it('erases every tenantless spelling when the legacy account stores an empty tenant', async () => {
const { userId, model } = await createResponses();
await User.updateOne({ _id: userId }, { $set: { tenantId: '' } });
const original = (await model.findOne({ userId, responseId: 'response-0' }).lean())!;
const { _id: _originalId, ...fields } = original;
await runAsSystem(async () => {
await model.create([
{ ...fields, tenantId: '', responseId: 'empty-tenant', requestKey: 'empty-tenant' },
{ ...fields, tenantId: null, responseId: 'null-tenant', requestKey: 'null-tenant' },
]);
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
expect(await model.countDocuments({ userId, tenantId: { $in: [null, ''] } })).toBe(0);
expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(1);
});
});
it('preserves the account on cleanup failure and allows a retry', async () => {
const { userId, model } = await createResponses();
jest
.spyOn(model.collection, 'deleteMany')
.mockRejectedValueOnce(new Error('response-store down'));
const deletion = jest.spyOn(User, 'deleteOne');
await expect(methods.deleteUserById(userId)).rejects.toThrow('response-store down');
expect(deletion).not.toHaveBeenCalled();
expect(await User.exists({ _id: userId })).not.toBeNull();
expect(await createResponseStorage(mongoose).read({ userId }, 'response-0')).not.toBeNull();
await methods.updateUser(userId, { name: 'Still usable' });
expect((await User.findById(userId).lean())?.name).toBe('Still usable');
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
});
it('uses the account tenant when deletion runs under system authority', async () => {
const { userId, model } = await createResponses();
await User.updateOne({ _id: userId }, { $set: { tenantId: 'another-tenant' } });
await runAsSystem(async () => {
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(0);
expect(await model.countDocuments({ userId, tenantId: null })).toBe(1);
});
});
});

View file

@ -16,6 +16,7 @@ import type {
} from '~/types';
import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types';
import type { CacheStore } from '~/types';
import { closeResponseScopes, eraseResponseRecords } from './responseScope';
import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search';
import { evictAuthUserDocsForUsers } from '~/utils/eviction';
import { signPayload } from '~/crypto';
@ -641,6 +642,11 @@ export function createUserMethods(
async function deleteUserById(userId: string): Promise<UserDeleteResult> {
try {
const User = mongoose.models.User;
const owner = await User.findById(userId).select('tenantId').lean<Pick<IUser, 'tenantId'>>();
if (owner) {
await closeResponseScopes(mongoose, { userId, tenantId: owner.tenantId });
await eraseResponseRecords(mongoose, { userId, tenantId: owner.tenantId });
}
await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId });
const result = await User.deleteOne({ _id: userId });
if (result.deletedCount === 0) {

View file

@ -1,4 +1,5 @@
import { createAgentQueuedTurnModel, createAgentQueuedTurnSequenceModel } from './queuedTurn';
import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';
import { createScheduleModel, createScheduleRunModel } from './schedule';
import { getTenantIndexMigrationHint } from '~/migrations/tenantIndexes';
@ -10,12 +11,15 @@ import { createAgentTriggerDeliveryModel } from './triggerDelivery';
import { createToolApprovalGrantModel } from './toolApprovalGrant';
import { createSkillUploadIntentModel } from './skillUploadIntent';
import { createSubagentContentModel } from './subagentContent';
import { createSkillSyncStatusModel } from './skillSyncStatus';
import { createConversationTagModel } from './conversationTag';
import { createCodeEnvironmentModel } from './codeEnvironment';
import { createSkillDeletionModel } from './skillDeletion';
import { createAgentCategoryModel } from './agentCategory';
import { createResponseScopeModel } from './responseScope';
import { createResourceLockModel } from './resourceLock';
import { createStoredResponseModel } from './response';
import { createChatProjectModel } from './chatProject';
import { createAgentApiKeyModel } from './agentApiKey';
import { createTransactionModel } from './transaction';
@ -57,6 +61,8 @@ import logger from '~/config/winston';
*/
export function createModels(mongoose: typeof import('mongoose')): {
User: ReturnType<typeof createUserModel>;
StoredResponse: ReturnType<typeof createStoredResponseModel>;
ResponseScope: ReturnType<typeof createResponseScopeModel>;
ToolApprovalGrant: ReturnType<typeof createToolApprovalGrantModel>;
Token: ReturnType<typeof createTokenModel>;
Session: ReturnType<typeof createSessionModel>;
@ -112,6 +118,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
} {
const models = {
User: createUserModel(mongoose),
StoredResponse: createStoredResponseModel(mongoose),
ResponseScope: createResponseScopeModel(mongoose),
ToolApprovalGrant: createToolApprovalGrantModel(mongoose),
Token: createTokenModel(mongoose),
Session: createSessionModel(mongoose),

View file

@ -0,0 +1,195 @@
import { Schema } from 'mongoose';
import type { Model } from 'mongoose';
import { applyTenantIsolation } from './plugins/tenantIsolation';
export interface ResponseOwner {
userId: string;
tenantId?: string;
}
export type StoredResponseStatus =
| 'queued'
| 'in_progress'
| 'requires_action'
| 'completed'
| 'failed'
| 'incomplete'
| 'cancelled';
export interface ResponseReviewCheckpoint {
actionId: string;
providerExecutionId: string;
expiresAt: number;
/** Private seed for the resumed output; never part of the public response view. */
partialResponseJSON: string;
/** Visible assistant token counter, separate from billed multi-model usage. */
visibleOutputTokens?: number;
/** Private reviewed graph/tool/checkpoint bindings retained after the job consumes its action. */
pendingActionJSON?: string;
/** Exact checkpointer settings captured by the paused run, independent of later config changes. */
checkpointConfigJSON?: string;
checkpointId?: string;
checkpointNs?: string;
}
export interface ResponseResumeIntent {
intentId: string;
actionId: string;
previousProviderExecutionId: string;
providerExecutionId: string;
fingerprint: string;
/** Validated, owner-submitted SDK resolution; not a credential or executable request. */
resolutionJSON: string;
/** Original validated browser decision, including edited arguments and remembered-approval scope. */
decisionJSON?: string;
state: 'reserved' | 'admitted';
reservedAt: number;
admittedAt?: number;
}
/** Server-written receipt after provider and trailing writes have definitively settled. */
export interface ResponseTerminalReceipt {
generationCreatedAt: number;
providerExecutionId: string;
ownerId?: string;
status: 'completed' | 'failed' | 'incomplete' | 'cancelled';
responseJSON: string;
checkpointNamespace?: string;
/** Server-pinned saver settings, including initial failures before a review exists. */
checkpointConfigJSON?: string;
}
/** Durable identity and dispatch/result journal for a background Responses request. */
export interface StoredResponseRecord {
expiredAt?: Date;
isTemporary?: boolean;
tenantId?: string;
userId: string;
responseId: string;
requestKey: string;
fingerprint: string;
agentId: string;
sourceKeyId?: string;
requestJSON: string;
conversationId: string;
parentMessageId: string;
status: StoredResponseStatus;
revision: number;
deliveryKey?: string;
generationCreatedAt?: number;
responseJSON?: string;
cancellationRequestedAt?: number;
review?: ResponseReviewCheckpoint;
resume?: ResponseResumeIntent;
pendingTerminal?: ResponseTerminalReceipt;
createdAt: Date;
updatedAt: Date;
}
export function createStoredResponseModel(
mongoose: typeof import('mongoose'),
): Model<StoredResponseRecord> {
const schema = new Schema<StoredResponseRecord>(
{
tenantId: String,
expiredAt: Date,
isTemporary: Boolean,
userId: { type: String, required: true },
responseId: { type: String, required: true },
requestKey: { type: String, required: true },
fingerprint: { type: String, required: true },
agentId: { type: String, required: true },
sourceKeyId: String,
requestJSON: { type: String, required: true },
conversationId: { type: String, required: true },
parentMessageId: { type: String, required: true },
status: {
type: String,
required: true,
enum: [
'queued',
'in_progress',
'requires_action',
'completed',
'failed',
'incomplete',
'cancelled',
],
},
revision: { type: Number, required: true, default: 0 },
deliveryKey: String,
generationCreatedAt: Number,
responseJSON: String,
cancellationRequestedAt: Number,
pendingTerminal: {
type: new Schema<ResponseTerminalReceipt>(
{
generationCreatedAt: { type: Number, required: true, min: 1 },
providerExecutionId: { type: String, required: true, maxlength: 128 },
ownerId: { type: String, maxlength: 128 },
status: {
type: String,
required: true,
enum: ['completed', 'failed', 'incomplete', 'cancelled'],
},
responseJSON: { type: String, required: true },
checkpointNamespace: String,
checkpointConfigJSON: String,
},
{ _id: false },
),
required: false,
},
review: {
type: new Schema<ResponseReviewCheckpoint>(
{
actionId: { type: String, required: true, maxlength: 256 },
providerExecutionId: { type: String, required: true, maxlength: 128 },
expiresAt: { type: Number, required: true, min: 1 },
partialResponseJSON: { type: String, required: true },
visibleOutputTokens: {
type: Number,
min: 0,
max: Number.MAX_SAFE_INTEGER,
validate: Number.isSafeInteger,
},
pendingActionJSON: String,
checkpointConfigJSON: String,
checkpointId: String,
checkpointNs: String,
},
{ _id: false },
),
required: false,
},
resume: {
type: new Schema<ResponseResumeIntent>(
{
intentId: { type: String, required: true, maxlength: 256 },
actionId: { type: String, required: true, maxlength: 256 },
previousProviderExecutionId: { type: String, required: true, maxlength: 128 },
providerExecutionId: { type: String, required: true, maxlength: 128 },
fingerprint: { type: String, required: true },
resolutionJSON: { type: String, required: true },
decisionJSON: String,
state: { type: String, required: true, enum: ['reserved', 'admitted'] },
reservedAt: { type: Number, required: true, min: 1 },
admittedAt: { type: Number, min: 1 },
},
{ _id: false },
),
required: false,
},
},
{ timestamps: true, minimize: false },
);
schema.index({ tenantId: 1, userId: 1, responseId: 1 }, { unique: true });
schema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
schema.index({ tenantId: 1, userId: 1, requestKey: 1 }, { unique: true });
schema.index({ tenantId: 1, userId: 1, conversationId: 1, createdAt: 1 });
schema.index({ userId: 1, conversationId: 1, tenantId: 1 });
applyTenantIsolation(schema);
return (
mongoose.models.StoredResponse ?? mongoose.model<StoredResponseRecord>('StoredResponse', schema)
);
}

View file

@ -0,0 +1,26 @@
import { Schema } from 'mongoose';
import type { Model } from 'mongoose';
interface ResponseScope {
_id: string;
closed: boolean;
revision: number;
expiredAt?: Date;
}
/** Payload-free admission anchors; closed scopes remain tombstones after erasure. */
export function createResponseScopeModel(
mongoose: typeof import('mongoose'),
): Model<ResponseScope> {
if (mongoose.models.ResponseScope) return mongoose.model<ResponseScope>('ResponseScope');
const schema = new Schema<ResponseScope>(
{
_id: { type: String, required: true },
closed: { type: Boolean, required: true, default: false },
revision: { type: Number, required: true, default: 0 },
expiredAt: Date,
},
{ versionKey: false },
);
return mongoose.model<ResponseScope>('ResponseScope', schema);
}

View file

@ -1,105 +1,76 @@
import logger from '~/config/winston';
export const CANCEL_RATE = 1.15;
const TRANSACTION_PROBE_COLLECTION = '__transaction_test__';
function unsupportedTransaction(error: unknown, mongoose: typeof import('mongoose')): boolean {
return (
error instanceof mongoose.mongo.MongoServerError &&
error.code === 20 &&
/Transaction numbers are only allowed on a replica set member or mongos/i.test(error.message)
);
}
async function probeTransaction(mongoose: typeof import('mongoose')): Promise<boolean> {
const session = await mongoose.startSession();
try {
const session = await mongoose.startSession();
try {
session.startTransaction();
await mongoose.connection.db
?.collection(TRANSACTION_PROBE_COLLECTION)
.findOne({}, { session });
await session.commitTransaction();
logger.debug('MongoDB transactions are supported');
return true;
} catch (transactionError: unknown) {
try {
await session.abortTransaction();
} catch (transactionError) {
/** best-effort abort */
logger.error(`[supportsTransactions] Error aborting transaction:`, transactionError);
}
logger.debug(
'MongoDB transactions not supported (transaction error):',
(transactionError as Error)?.message || 'Unknown error',
);
return false;
} finally {
await session.endSession();
}
session.startTransaction();
const db = mongoose.connection.db;
if (!db) throw new Error('Transaction capability probe requires a connected database');
await db.collection(TRANSACTION_PROBE_COLLECTION).findOne({}, { session });
await session.commitTransaction();
return true;
} catch (error) {
logger.debug(
'MongoDB transactions not supported (session error):',
(error as Error)?.message || 'Unknown error',
);
return false;
try {
await session.abortTransaction();
} catch {
logger.debug('[supportsTransactions] Probe transaction abort failed');
}
if (unsupportedTransaction(error, mongoose)) return false;
throw error;
} finally {
await session.endSession();
}
}
/**
* Checks if the connected MongoDB deployment supports transactions
* This requires a MongoDB replica set configuration
*
* Amazon DocumentDB rejects a transaction that touches a collection which does
* not exist ("Feature not supported: non-existent collection in transaction"),
* so a failed first probe materializes the canary and probes once more —
* otherwise engines that fully support transactions would report unsupported
* and silently drop every caller to the non-transactional path. MongoDB allows
* transactional reads of missing collections, so deployments that pass the
* first probe never pay the create and never gain the canary collection. A
* failed create is logged rather than swallowed: on a hardened role it is the
* only visible explanation for a wrong "unsupported" verdict.
*
* @returns True if transactions are supported, false otherwise
*/
/** Only confirmed standalone incompatibility is false; operational failures stay retryable. */
export const supportsTransactions = async (
mongoose: typeof import('mongoose'),
): Promise<boolean> => {
if (await probeTransaction(mongoose)) {
return true;
}
try {
await mongoose.connection.db?.createCollection(TRANSACTION_PROBE_COLLECTION);
return await probeTransaction(mongoose);
} catch (error) {
const message = (error as Error)?.message ?? '';
if (!/already exists|NamespaceExists/i.test(message)) {
logger.warn(
`[supportsTransactions] Could not create the "${TRANSACTION_PROBE_COLLECTION}" probe collection; ` +
'if this deployment is Amazon DocumentDB, the transaction-support verdict below may be a false negative:',
error,
);
// DocumentDB requires the probe collection to exist before a transaction can read it.
if (
!(error instanceof mongoose.mongo.MongoServerError) ||
!/non-existent collection in transaction/i.test(error.message)
)
throw error;
const db = mongoose.connection.db;
if (!db) throw error;
try {
await db.createCollection(TRANSACTION_PROBE_COLLECTION);
} catch (creationError) {
if (!(creationError instanceof mongoose.mongo.MongoServerError) || creationError.code !== 48)
throw creationError;
}
return probeTransaction(mongoose);
}
return probeTransaction(mongoose);
};
let inflightProbe: Promise<boolean> | null = null;
const inflightProbes = new WeakMap<import('mongoose').Connection, Promise<boolean>>();
/**
* Gets whether the current MongoDB deployment supports transactions
* Caches the result for performance
*
* Concurrent first callers share one in-flight probe instead of each paying
* the session/transaction round trips before the caller-side cache fills.
*
* @returns True if transactions are supported, false otherwise
*/
/** Share first probes per connection. Callers cache only a successful capability verdict. */
export const getTransactionSupport = async (
mongoose: typeof import('mongoose'),
transactionSupportCache: boolean | null,
): Promise<boolean> => {
if (transactionSupportCache !== null) {
return transactionSupportCache;
if (transactionSupportCache !== null) return transactionSupportCache;
const connection = mongoose.connection;
let probe = inflightProbes.get(connection);
if (!probe) {
probe = supportsTransactions(mongoose).finally(() => inflightProbes.delete(connection));
inflightProbes.set(connection, probe);
}
if (inflightProbe == null) {
inflightProbe = supportsTransactions(mongoose).finally(() => {
inflightProbe = null;
});
}
return await inflightProbe;
return probe;
};