mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 05:20:55 +00:00
🗄️ feat: Persist and Expire Background Responses (#17594)
* 🗄️ feat: Persist Background Responses
* fix: unify response retirement and cleanup identity
* fix: preserve immutable conversation identity and retry capability probes
* test: Await Terminal Replay Instead of a Live Subscription
* test: Keep Delayed Queue Admission Fixtures Consistent
* test: Scope Upload Assertions to the Composer Tray
* test: Observe Paused Writes and Scope Editor Controls
* ci: Allow Complete Database Mutation Reports
This commit is contained in:
parent
b7fa9c1e71
commit
1458e841e4
31 changed files with 3321 additions and 104 deletions
3
.github/workflows/change-quality.yml
vendored
3
.github/workflows/change-quality.yml
vendored
|
|
@ -74,7 +74,8 @@ jobs:
|
|||
needs: [select, verify]
|
||||
if: needs.select.outputs.hasChanges == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
# Whole edited database functions can produce thousands of mutation cases.
|
||||
timeout-minutes: ${{ matrix.check == 'mutation' && 240 || 45 }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: source
|
||||
|
|
|
|||
|
|
@ -341,7 +341,9 @@ test.describe('core chat loop', () => {
|
|||
|
||||
await uploadProviderFile(page, csvFixture);
|
||||
|
||||
await expect(page.getByRole('button', { name: filename })).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId('composer-tray').getByRole('button', { name: filename, exact: true }),
|
||||
).toBeVisible();
|
||||
|
||||
const input = page.getByRole('textbox', { name: 'Message input' });
|
||||
await input.click();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
import type { Page, Route } from '@playwright/test';
|
||||
import type { AgentDetail } from './agents.helpers';
|
||||
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
|
||||
|
|
@ -27,6 +28,7 @@ const messageTurns = (page: Page) => messagesView(page).locator('.message-render
|
|||
|
||||
const RECEIPTS_ROUTE = /\/api\/agents\/chat\/queued-turns\?/;
|
||||
const STATUS_ROUTE = /\/api\/agents\/chat\/status\//;
|
||||
const MESSAGES_ROUTE = /\/api\/messages\/[^/?]+(?:\?|$)/;
|
||||
|
||||
type ReceiptsBody = { queuedTurns?: Array<{ status?: string }> };
|
||||
type StatusBody = { active?: boolean; resumeState?: { userMessage?: { text?: string } } };
|
||||
|
|
@ -39,8 +41,9 @@ type StatusBody = { active?: boolean; resumeState?: { userMessage?: { text?: str
|
|||
* only through the receipt projection and the status it re-arms. While held
|
||||
* back, the receipts stay frozen at their pre-admission snapshot and a status
|
||||
* that describes the successor reads as inactive, so the next user turn can
|
||||
* only come from the completion event itself. Every other request, including
|
||||
* the predecessor's own terminal status, passes through untouched.
|
||||
* only come from the completion event itself. History also withholds the admitted
|
||||
* successor and its descendants, so a backfill cannot bypass the hold. Other requests,
|
||||
* including the predecessor's own terminal status, pass through untouched.
|
||||
*/
|
||||
function holdBackSuccessor(page: Page, successorText: string) {
|
||||
let holding = false;
|
||||
|
|
@ -70,10 +73,39 @@ function holdBackSuccessor(page: Page, successorText: string) {
|
|||
json: describesSuccessor ? { active: false } : body,
|
||||
});
|
||||
};
|
||||
const messages = async (route: Route) => {
|
||||
const response = await route.fetch();
|
||||
if (!holding || response.status() !== 200 || route.request().method() !== 'GET') {
|
||||
return route.fulfill({ response });
|
||||
}
|
||||
const body = (await response.json()) as TMessage[];
|
||||
const hidden = new Set<string>();
|
||||
const children = new Map<string, string[]>();
|
||||
for (const message of body) {
|
||||
if (message.isCreatedByUser && message.text === successorText) hidden.add(message.messageId);
|
||||
const parent = message.parentMessageId ?? '';
|
||||
const siblings = children.get(parent) ?? [];
|
||||
siblings.push(message.messageId);
|
||||
children.set(parent, siblings);
|
||||
}
|
||||
const pending = [...hidden];
|
||||
for (const id of pending) {
|
||||
for (const child of children.get(id) ?? []) {
|
||||
if (hidden.has(child)) continue;
|
||||
hidden.add(child);
|
||||
pending.push(child);
|
||||
}
|
||||
}
|
||||
return route.fulfill({
|
||||
response,
|
||||
json: body.filter((message) => !hidden.has(message.messageId)),
|
||||
});
|
||||
};
|
||||
return {
|
||||
arm: async () => {
|
||||
await page.route(RECEIPTS_ROUTE, receipts);
|
||||
await page.route(STATUS_ROUTE, status);
|
||||
await page.route(MESSAGES_ROUTE, messages);
|
||||
},
|
||||
hold: () => {
|
||||
holding = true;
|
||||
|
|
@ -82,6 +114,7 @@ function holdBackSuccessor(page: Page, successorText: string) {
|
|||
holding = false;
|
||||
await page.unroute(RECEIPTS_ROUTE, receipts);
|
||||
await page.unroute(STATUS_ROUTE, status);
|
||||
await page.unroute(MESSAGES_ROUTE, messages);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -166,13 +199,14 @@ test.describe('server-queued follow-up reveal', () => {
|
|||
const predecessor = messageTurns(page).nth(3);
|
||||
await expect(predecessor).toContainText(SLOW_REPLY_LAST_CHUNK, { timeout: 60000 });
|
||||
|
||||
const pending = page.getByTestId('pending-turn');
|
||||
await expect(pending).toBeVisible({ timeout: 5000 });
|
||||
const queuedTurn = messageTurns(page).nth(4);
|
||||
await expect(queuedTurn).toContainText(queueText, { timeout: 5000 });
|
||||
await expect(queuedTurn.locator('.user-turn')).toBeVisible();
|
||||
/** One visible copy: actions follow the user turn, not a duplicate chip
|
||||
* below the composer. Other queued turns still use their ordinary rows. */
|
||||
await expect(row).toHaveCount(0);
|
||||
const pending = page.getByTestId('pending-turn');
|
||||
await expect(pending).toContainText('Starting as the next turn');
|
||||
await expect(pending.getByRole('button', { name: 'Send now' })).toHaveCount(0);
|
||||
await expect(pending.getByRole('button', { name: 'Remove message' })).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -507,7 +507,7 @@ test.describe('compaction rerun controls', () => {
|
|||
await page.locator(`[id="edit-${chainedId}"]`).click();
|
||||
const editor = page.getByTestId('message-text-editor');
|
||||
await expect(editor).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Save' })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Save', exact: true })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Rerun', exact: true })).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Update & rerun' })).toHaveCount(0);
|
||||
/* ...and the footer says why the action it usually carries is gone. */
|
||||
|
|
@ -545,7 +545,7 @@ test.describe('compaction rerun controls', () => {
|
|||
await page.locator(`[id="edit-${rootReplyId}"]`).click();
|
||||
const editor = page.getByTestId('message-text-editor');
|
||||
await expect(editor).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Save' })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Save', exact: true })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Rerun', exact: true })).toHaveCount(0);
|
||||
await expect(
|
||||
page.getByText('Rerunning replays the user message before a response', { exact: false }),
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ async function attachMarkdown(page: Page) {
|
|||
expect(response.ok()).toBeTruthy();
|
||||
const file = (await response.json()) as UploadedFile & { filepath?: string };
|
||||
expect(file.llmDeliveryPath).toBe('provider');
|
||||
await expect(page.getByRole('button', { name })).toBeVisible({ timeout: 15000 });
|
||||
await expect(
|
||||
page.getByTestId('composer-tray').getByRole('button', { name, exact: true }),
|
||||
).toBeVisible({ timeout: 15000 });
|
||||
return { name, file };
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,7 +87,9 @@ test.describe('Google native media in the context budget', () => {
|
|||
});
|
||||
expect(upload.ok()).toBeTruthy();
|
||||
expect(((await upload.json()) as UploadedFile).llmDeliveryPath).toBe('provider');
|
||||
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
|
||||
await expect(
|
||||
page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }),
|
||||
).toBeVisible({ timeout: 15000 });
|
||||
|
||||
const report = await reportContextTokens(page);
|
||||
expect(report.provider).toBe('google');
|
||||
|
|
@ -107,7 +109,9 @@ test.describe('Google native media in the context budget', () => {
|
|||
content: syntheticPdf(),
|
||||
});
|
||||
expect(upload.ok()).toBeTruthy();
|
||||
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
|
||||
await expect(
|
||||
page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }),
|
||||
).toBeVisible({ timeout: 15000 });
|
||||
|
||||
const first = await reportContextTokens(page);
|
||||
expect(first.latest).toBeGreaterThan(ONE_PAGE_TOKENS);
|
||||
|
|
|
|||
|
|
@ -148,7 +148,11 @@ test.describe('shared links', () => {
|
|||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
|
||||
await uploadProviderFile(page, fileFixture);
|
||||
await expect(page.getByRole('button', { name: fileFixture.name, exact: true })).toBeVisible();
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('composer-tray')
|
||||
.getByRole('button', { name: fileFixture.name, exact: true }),
|
||||
).toBeVisible();
|
||||
|
||||
const response = await sendMessage(page, userMessage);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
|
|
|||
|
|
@ -125,8 +125,8 @@ test.describe('skill file authoring tools', () => {
|
|||
await expect(page.getByRole('heading', { level: 1, name: skillName })).toBeVisible();
|
||||
await expect(page.getByText(EDITED_DESCRIPTION)).toBeVisible();
|
||||
|
||||
await page.getByRole('button', { name: 'Delete' }).click();
|
||||
await page.getByRole('dialog').getByRole('button', { name: 'Delete' }).click();
|
||||
await page.getByTestId('app-pane').getByRole('button', { name: 'Delete', exact: true }).click();
|
||||
await page.getByRole('dialog').getByRole('button', { name: 'Delete', exact: true }).click();
|
||||
await expect(page).toHaveURL((url) => url.pathname === '/skills');
|
||||
const token = await getAccessToken(page);
|
||||
await expect.poll(async () => findSkill(page, skillName, token)).toBeNull();
|
||||
|
|
|
|||
|
|
@ -93,7 +93,9 @@ test.describe('unified file upload', () => {
|
|||
expect(((await response.json()) as UploadedFile).llmDeliveryPath).toBe('provider');
|
||||
|
||||
// (a) shows as an attachment chip in the composer before sending.
|
||||
await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 });
|
||||
await expect(
|
||||
page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }),
|
||||
).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// (b) reaches the model input: the mock LLM echoes a pass marker only when the
|
||||
// provider file is present in the request content (see e2e/setup/fake-model.js).
|
||||
|
|
|
|||
|
|
@ -400,12 +400,12 @@ describe('terminal projection boundary', () => {
|
|||
delivered.push(event);
|
||||
},
|
||||
);
|
||||
expect(subscription).not.toBeNull();
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
// A completed replay may retire its subscription before attachment returns.
|
||||
await waitFor(() => delivered.length > 0, 'legacy terminal replay');
|
||||
|
||||
expect(delivered).toHaveLength(1);
|
||||
expectSafeTerminalEvent(delivered[0]);
|
||||
subscription!.unsubscribe();
|
||||
subscription?.unsubscribe();
|
||||
} finally {
|
||||
await replica.destroy();
|
||||
await replicaSubscriber.quit().catch(() => undefined);
|
||||
|
|
|
|||
|
|
@ -1140,6 +1140,12 @@ const remoteApiAuthSchema = z.object({
|
|||
|
||||
const remoteApiSchema = z.object({
|
||||
auth: remoteApiAuthSchema.optional(),
|
||||
background: z
|
||||
.object({
|
||||
enabled: z.boolean().default(false),
|
||||
acknowledgementTimeoutMs: z.number().int().min(100).max(60_000).default(10_000),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const managementClientBindingSchema = z
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ export * from './roles';
|
|||
/* types (exports schemas from `./types` as they contain needed in other defs) */
|
||||
export * from './types';
|
||||
export * from './types/agents';
|
||||
export * from './types/responseAccounting';
|
||||
export * from './types/assistants';
|
||||
export * from './types/content';
|
||||
export * from './types/tools';
|
||||
|
|
@ -112,4 +113,6 @@ export * from './types/scheduleConsent';
|
|||
|
||||
export * from './approval';
|
||||
|
||||
export * from './types/responseReview';
|
||||
|
||||
export * from './codeSetup';
|
||||
|
|
|
|||
45
packages/data-provider/src/types/responseAccounting.ts
Normal file
45
packages/data-provider/src/types/responseAccounting.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
/** An application-billing amount is not a provider invoice or list-price estimate. */
|
||||
export type AccountingBasis = 'provider_reported' | 'application_billing';
|
||||
export type AccountingModelSource =
|
||||
| 'provider_reported'
|
||||
| 'runtime_reported'
|
||||
| 'configured'
|
||||
| 'unknown';
|
||||
|
||||
export interface ResponseAccountingCall {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
modelSource: AccountingModelSource;
|
||||
usageType:
|
||||
| 'message'
|
||||
| 'summarization'
|
||||
| 'subagent'
|
||||
| 'sequential'
|
||||
| 'activity-label'
|
||||
| 'activity-phase'
|
||||
| 'reasoning-label';
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cacheReadTokens: number;
|
||||
cacheWriteTokens: number;
|
||||
amount?: string;
|
||||
basis?: AccountingBasis;
|
||||
reason: 'priced' | 'pricing_unavailable' | 'usage_unavailable' | 'invalid_usage';
|
||||
}
|
||||
|
||||
/** Separate from token usage: partial costs must never appear as a run's total. */
|
||||
export interface ResponseAccounting {
|
||||
status: 'available' | 'unavailable';
|
||||
currency: 'USD';
|
||||
amount?: string;
|
||||
basis?: AccountingBasis;
|
||||
coverage: 'complete' | 'partial' | 'none';
|
||||
reason:
|
||||
| 'priced'
|
||||
| 'pricing_unavailable'
|
||||
| 'usage_unavailable'
|
||||
| 'partial_coverage'
|
||||
| 'mixed_bases'
|
||||
| 'conflicting_usage';
|
||||
calls: ResponseAccountingCall[];
|
||||
}
|
||||
55
packages/data-provider/src/types/responseReview.ts
Normal file
55
packages/data-provider/src/types/responseReview.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import { z } from 'zod';
|
||||
import type { Agents } from './agents';
|
||||
|
||||
export const responseReviewDecisionSchema = z
|
||||
.object({
|
||||
actionId: z.string().min(1).max(256),
|
||||
answer: z.string().optional(),
|
||||
answers: z.record(z.string(), z.string()).optional(),
|
||||
decisions: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
tool_call_id: z.string(),
|
||||
decision: z.enum(['approve', 'reject', 'edit', 'respond']),
|
||||
editedArguments: z.record(z.string(), z.unknown()).optional(),
|
||||
responseText: z.string().optional(),
|
||||
reason: z.string().optional(),
|
||||
scope: z.enum(['once', 'session', 'always']).optional(),
|
||||
})
|
||||
.strict(),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type ResponseReviewDecision = z.infer<typeof responseReviewDecisionSchema>;
|
||||
export const responseReviewDecisionReceiptSchema = z.object({
|
||||
responseId: z.string().min(1),
|
||||
actionId: z.string().min(1),
|
||||
intentId: z.string().min(1),
|
||||
status: z.enum(['queued', 'settled']),
|
||||
});
|
||||
export type ResponseReviewDecisionReceipt = z.infer<typeof responseReviewDecisionReceiptSchema>;
|
||||
export const responseCancelReceiptSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
status: z.enum(['queued', 'in_progress', 'completed', 'failed', 'incomplete', 'cancelled']),
|
||||
});
|
||||
export type ResponseCancelReceipt = z.infer<typeof responseCancelReceiptSchema>;
|
||||
|
||||
/** Owner-authenticated browser view; OAuth URLs must never be delivered to a channel. */
|
||||
export interface ResponseReviewView {
|
||||
/** Missing/mismatched execution is not proof that provider work stopped. */
|
||||
recovery?: 'execution_state_unavailable';
|
||||
responseId: string;
|
||||
status: 'queued' | 'in_progress' | 'completed' | 'failed' | 'incomplete' | 'cancelled';
|
||||
oauth: Array<{
|
||||
actionId: string;
|
||||
toolName: string;
|
||||
authURL: string;
|
||||
expiresAt: number;
|
||||
}>;
|
||||
pendingAction?: Pick<Agents.PendingAction, 'actionId' | 'payload' | 'expiresAt'>;
|
||||
pendingDecision?: { intentId: string; decision: ResponseReviewDecision };
|
||||
canSubmitDecision?: boolean;
|
||||
}
|
||||
|
|
@ -11,6 +11,22 @@ export * from './crypto';
|
|||
export * from './schema';
|
||||
export * from './utils';
|
||||
export { createModels } from './models';
|
||||
export { createResponseStorage } from './methods/response';
|
||||
export type {
|
||||
ResponseOwner,
|
||||
ResponseStorage,
|
||||
ResponseCleanupIdentity,
|
||||
ResponseReservation,
|
||||
ResponseReservationResult,
|
||||
ResponseMutationResult,
|
||||
ResponseReviewCheckpoint,
|
||||
ResponseResumeIntent,
|
||||
ResponseResumeReservation,
|
||||
ResponseResumeResult,
|
||||
ResponseTerminalReceipt,
|
||||
StoredResponseRecord,
|
||||
StoredResponseStatus,
|
||||
} from './methods/response';
|
||||
export {
|
||||
createMethods,
|
||||
createSkillDeletionPersistenceAdapter,
|
||||
|
|
@ -103,6 +119,7 @@ export {
|
|||
AUDIT_ACTION_CATEGORY,
|
||||
} from './types/admin';
|
||||
export { GENESIS_HASH, PLATFORM_CHAIN_KEY } from './schema/auditLog';
|
||||
|
||||
export { default as logger, baseLogFormat } from './config/winston';
|
||||
export { default as meiliLogger } from './config/meiliLogger';
|
||||
export { jsonTruncateFormat, redactMessage } from './config/parsers';
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ import type {
|
|||
import { ConversationMethods, createConversationMethods } from './conversation';
|
||||
import { tenantStorage, runAsSystem } from '~/config/tenantContext';
|
||||
import { createChatProjectMethods } from './chatProject';
|
||||
import { createResponseStorage } from './response';
|
||||
import { waitForPause } from './test-helpers';
|
||||
import { createModels } from '../models';
|
||||
|
||||
jest.mock('~/config/winston', () => ({
|
||||
|
|
@ -1141,13 +1143,13 @@ describe('Conversation Operations', () => {
|
|||
},
|
||||
);
|
||||
|
||||
const archive = saveConvo(
|
||||
{ userId: 'user123' },
|
||||
{ conversationId, isArchived: true },
|
||||
{ preserveUpdatedAt: true, noUpsert: true },
|
||||
);
|
||||
try {
|
||||
const archive = saveConvo(
|
||||
{ userId: 'user123' },
|
||||
{ conversationId, isArchived: true },
|
||||
{ preserveUpdatedAt: true, noUpsert: true },
|
||||
);
|
||||
await archiveWriteReached;
|
||||
await waitForPause(archiveWriteReached, archive);
|
||||
|
||||
const unarchived = await saveConvo(
|
||||
{ userId: 'user123' },
|
||||
|
|
@ -1166,6 +1168,7 @@ describe('Conversation Operations', () => {
|
|||
} finally {
|
||||
releaseArchiveWrite();
|
||||
writeSpy.mockRestore();
|
||||
await Promise.allSettled([archive]);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -3848,6 +3851,150 @@ describe('Conversation Operations', () => {
|
|||
});
|
||||
|
||||
describe('deleteConvos', () => {
|
||||
describe('stored responses on standalone Mongo', () => {
|
||||
beforeEach(async () => {
|
||||
await runAsSystem(async () => {
|
||||
await mongoose.models.StoredResponse.deleteMany({});
|
||||
await mongoose.models.ResponseScope.deleteMany({});
|
||||
});
|
||||
});
|
||||
|
||||
async function seedResponse(conversationId: string, tenantId?: string) {
|
||||
return runAsSystem(() =>
|
||||
mongoose.models.StoredResponse.create({
|
||||
userId: 'user123',
|
||||
tenantId,
|
||||
conversationId,
|
||||
responseId: uuidv4(),
|
||||
requestKey: uuidv4(),
|
||||
fingerprint: 'request',
|
||||
agentId: 'agent',
|
||||
requestJSON: '{"input":"private"}',
|
||||
responseJSON: '{"output":"private"}',
|
||||
parentMessageId: 'root',
|
||||
status: 'completed',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
it('rejects background reservation on standalone Mongo before storing private state', async () => {
|
||||
const result = await createResponseStorage(mongoose).reserve({
|
||||
userId: 'user123',
|
||||
responseId: uuidv4(),
|
||||
requestKey: uuidv4(),
|
||||
fingerprint: 'request',
|
||||
agentId: 'agent',
|
||||
requestJSON: '{"input":"private"}',
|
||||
conversationId: uuidv4(),
|
||||
parentMessageId: 'root',
|
||||
});
|
||||
expect(result).toEqual({ ok: false, code: 'response_unavailable' });
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0);
|
||||
expect(await mongoose.models.ResponseScope.countDocuments({})).toBe(0);
|
||||
});
|
||||
|
||||
it('erases only the selected tenant and keeps ordinary deletion nontransactional', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await Conversation.create({
|
||||
conversationId,
|
||||
user: 'user123',
|
||||
tenantId: 'tenant-a',
|
||||
endpoint: EModelEndpoint.agents,
|
||||
});
|
||||
await seedResponse(conversationId, 'tenant-a');
|
||||
await seedResponse(conversationId, 'tenant-b');
|
||||
const session = jest.spyOn(mongoose, 'startSession');
|
||||
await tenantStorage.run({ tenantId: 'tenant-a' }, () =>
|
||||
deleteConvos('user123', { conversationId }),
|
||||
);
|
||||
expect(session).not.toHaveBeenCalled();
|
||||
session.mockRestore();
|
||||
await runAsSystem(async () => {
|
||||
expect(
|
||||
await mongoose.models.StoredResponse.countDocuments({ tenantId: 'tenant-a' }),
|
||||
).toBe(0);
|
||||
expect(
|
||||
await mongoose.models.StoredResponse.countDocuments({ tenantId: 'tenant-b' }),
|
||||
).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
it('retains payloads when an earlier hook fails but keeps deletion intent closed until retry', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await Conversation.create({
|
||||
conversationId,
|
||||
user: 'user123',
|
||||
endpoint: EModelEndpoint.agents,
|
||||
});
|
||||
const response = await seedResponse(conversationId);
|
||||
await expect(
|
||||
deleteConvos(
|
||||
'user123',
|
||||
{ conversationId },
|
||||
{
|
||||
beforeDelete: async () => {
|
||||
throw new Error('drain failed');
|
||||
},
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('drain failed');
|
||||
expect(await Conversation.exists({ conversationId })).not.toBeNull();
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(1);
|
||||
expect(
|
||||
await createResponseStorage(mongoose).read({ userId: 'user123' }, response.responseId),
|
||||
).not.toBeNull();
|
||||
await deleteConvos('user123', { conversationId });
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
|
||||
});
|
||||
|
||||
it('recovers private response cleanup after the root and its messages are already gone', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await seedResponse(conversationId);
|
||||
const result = await deleteConvos('user123', { conversationId });
|
||||
expect(result.conversationIds).toContain(conversationId);
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
|
||||
});
|
||||
|
||||
it('retries irreversible response erasure when the subsequent parent deletion fails', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await Conversation.create({
|
||||
conversationId,
|
||||
user: 'user123',
|
||||
endpoint: EModelEndpoint.agents,
|
||||
});
|
||||
await seedResponse(conversationId);
|
||||
const deletion = jest
|
||||
.spyOn(Conversation.collection, 'deleteMany')
|
||||
.mockRejectedValueOnce(new Error('parent delete failed'));
|
||||
await expect(deleteConvos('user123', { conversationId })).rejects.toThrow(
|
||||
'parent delete failed',
|
||||
);
|
||||
deletion.mockRestore();
|
||||
expect(await Conversation.exists({ conversationId })).not.toBeNull();
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
|
||||
await deleteConvos('user123', { conversationId });
|
||||
expect(await Conversation.exists({ conversationId })).toBeNull();
|
||||
});
|
||||
|
||||
it('allows concurrent erasers to finish without reopening admission', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await Conversation.create({
|
||||
conversationId,
|
||||
user: 'user123',
|
||||
endpoint: EModelEndpoint.agents,
|
||||
});
|
||||
const response = await seedResponse(conversationId);
|
||||
await Promise.all([
|
||||
deleteConvos('user123', { conversationId }, { allowEmpty: true }),
|
||||
deleteConvos('user123', { conversationId }, { allowEmpty: true }),
|
||||
]);
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0);
|
||||
expect(
|
||||
await createResponseStorage(mongoose).read({ userId: 'user123' }, response.responseId),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it('retires queued-turn work before each conversation deletion wave', async () => {
|
||||
const conversationId = uuidv4();
|
||||
await Conversation.create({
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import type {
|
|||
IMessage,
|
||||
ISharedLink,
|
||||
ISubagentThreadReservation,
|
||||
IUser,
|
||||
} from '~/types';
|
||||
import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';
|
||||
import type { MessageMethods } from './message';
|
||||
|
|
@ -39,6 +40,13 @@ import {
|
|||
MAX_AGENT_EVENT_ACTOR_SUMMARY_LENGTH,
|
||||
MAX_AGENT_EVENT_ACTOR_TOOL_NAME_LENGTH,
|
||||
} from '~/types/convo';
|
||||
import {
|
||||
closeResponseScopes,
|
||||
eraseResponseRecords,
|
||||
createResponseRetention,
|
||||
discoverResponseOwners,
|
||||
createResponseConversationReplacement,
|
||||
} from './responseScope';
|
||||
import {
|
||||
activeExpirationFilter,
|
||||
buildRetentionVisibilityFilter,
|
||||
|
|
@ -49,6 +57,7 @@ import {
|
|||
updateChatProjectLastConversationForUser,
|
||||
} from './chatProject';
|
||||
import { createChatExpirationDate, createTempChatExpirationDate } from '~/utils/tempChatRetention';
|
||||
import { getTenantId, SYSTEM_TENANT_ID, runAsSystem } from '~/config/tenantContext';
|
||||
import { isAgentFadingTier, isAgentFadingTierEntries } from '~/utils/fading';
|
||||
import { isCompactionSemanticIndexProjection } from '~/types/compaction';
|
||||
import { withoutMeiliIndexing } from '~/models/plugins/mongoMeili';
|
||||
|
|
@ -668,6 +677,8 @@ export function createConversationMethods(
|
|||
mongoose: typeof import('mongoose'),
|
||||
deps?: ConversationMethodDeps,
|
||||
): ConversationMethods {
|
||||
const retainResponses = createResponseRetention(mongoose);
|
||||
const prepareResponseReplacement = createResponseConversationReplacement(mongoose);
|
||||
let legacyReceiptExpiryCursor: Types.ObjectId | undefined;
|
||||
|
||||
/**
|
||||
|
|
@ -2336,12 +2347,13 @@ export function createConversationMethods(
|
|||
|
||||
const Conversation = mongoose.models.Conversation as Model<IConversation>;
|
||||
const stored = await Conversation.findOne({ conversationId, user: userId })
|
||||
.select({ _id: 1, isTemporary: 1, expiredAt: 1, tags: 1 })
|
||||
.select({ _id: 1, isTemporary: 1, expiredAt: 1, tags: 1, tenantId: 1 })
|
||||
.lean<{
|
||||
_id: Types.ObjectId;
|
||||
isTemporary?: boolean;
|
||||
expiredAt?: Date | null;
|
||||
tags?: string[];
|
||||
tenantId?: string;
|
||||
} | null>();
|
||||
if (!stored) {
|
||||
return;
|
||||
|
|
@ -2357,6 +2369,7 @@ export function createConversationMethods(
|
|||
}
|
||||
}
|
||||
|
||||
await retainResponses({ userId, tenantId: stored.tenantId }, conversationId, expiredAt);
|
||||
let stampedDeadline = stored.expiredAt != null;
|
||||
if (stored.isTemporary !== true) {
|
||||
/**
|
||||
|
|
@ -2580,6 +2593,37 @@ export function createConversationMethods(
|
|||
update.expiredAt = null;
|
||||
}
|
||||
|
||||
const responseDeadline =
|
||||
update.expiredAt instanceof Date ? update.expiredAt : retentionOnInsert?.expiredAt;
|
||||
const replacingConversationId = newConversationId && newConversationId !== conversationId;
|
||||
if (responseDeadline || replacingConversationId) {
|
||||
const stored = await Conversation.findOne({ user: userId, conversationId })
|
||||
.select('tenantId expiredAt')
|
||||
.lean<Pick<IConversation, 'tenantId' | 'expiredAt'>>();
|
||||
const contextTenant = getTenantId();
|
||||
const responseOwner = {
|
||||
userId,
|
||||
tenantId:
|
||||
stored?.tenantId ?? (contextTenant === SYSTEM_TENANT_ID ? undefined : contextTenant),
|
||||
};
|
||||
// Conversation IDs are shared foreign keys for messages, approvals, jobs
|
||||
// and checkpoints. newConversationId assigns an initial ID, never renames
|
||||
// a persisted conversation independently of those resources.
|
||||
if (
|
||||
replacingConversationId &&
|
||||
(stored || !(await prepareResponseReplacement(responseOwner, conversationId)))
|
||||
) {
|
||||
return { message: 'Conversation ID cannot be replaced after persistence.' };
|
||||
}
|
||||
const deadline = responseDeadline ?? stored?.expiredAt;
|
||||
if (deadline)
|
||||
await retainResponses(
|
||||
responseOwner,
|
||||
newConversationId || conversationId,
|
||||
stored?.expiredAt && stored.expiredAt < deadline ? stored.expiredAt : deadline,
|
||||
);
|
||||
}
|
||||
|
||||
const createdAtOnInsert =
|
||||
metadata?.createdAtOnInsert instanceof Date &&
|
||||
!Number.isNaN(metadata.createdAtOnInsert.getTime())
|
||||
|
|
@ -2652,6 +2696,9 @@ export function createConversationMethods(
|
|||
const baseFilter = {
|
||||
conversationId,
|
||||
user: userId,
|
||||
// Initial ID assignment must never match a source inserted after the
|
||||
// absence check. Upsert may create the new ID, but cannot rename that row.
|
||||
...(replacingConversationId ? { _id: { $exists: false } } : {}),
|
||||
...(metadata?.titleSource === 'generated' && {
|
||||
titleSetByUser: { $ne: true },
|
||||
title: { $in: [null, '', 'New Chat'] },
|
||||
|
|
@ -2773,6 +2820,11 @@ export function createConversationMethods(
|
|||
interfaceConfig,
|
||||
conversation.isTemporary === true,
|
||||
);
|
||||
await retainResponses(
|
||||
{ userId, tenantId: conversation.tenantId },
|
||||
conversationId,
|
||||
deadline,
|
||||
);
|
||||
const result = await Conversation.updateOne(
|
||||
{
|
||||
_id: conversation._id,
|
||||
|
|
@ -4046,12 +4098,40 @@ export function createConversationMethods(
|
|||
.select('conversationId tenantId chatProjectId tags')
|
||||
.lean<DeletionConversation[]>();
|
||||
const recoveryConversationIds: string[] = [];
|
||||
const recoveryResponseScopes: DeletionConversation[] = [];
|
||||
const responseGroups = (wave: DeletionConversation[]) => {
|
||||
const groups = new Map<string | undefined, string[]>();
|
||||
for (const conversation of wave) {
|
||||
const tenantId = conversation.tenantId || undefined;
|
||||
const ids = groups.get(tenantId) ?? [];
|
||||
ids.push(conversation.conversationId);
|
||||
groups.set(tenantId, ids);
|
||||
}
|
||||
return groups;
|
||||
};
|
||||
const closeResponses = async (wave: DeletionConversation[]) => {
|
||||
for (const [tenantId, ids] of responseGroups(wave))
|
||||
await closeResponseScopes(mongoose, { userId: user, tenantId }, ids);
|
||||
};
|
||||
const eraseResponses = async (wave: DeletionConversation[]) => {
|
||||
for (const [tenantId, ids] of responseGroups(wave))
|
||||
await eraseResponseRecords(mongoose, { userId: user, tenantId }, ids);
|
||||
};
|
||||
if (!conversations.length && typeof filter.conversationId === 'string') {
|
||||
// Authorized delayed reservations retain the globally unique user's actual tenant.
|
||||
// Fence it before probing legacy payloads, including when no response exists yet.
|
||||
const owner = isValidObjectIdString(user)
|
||||
? await (mongoose.models.User as Model<IUser>).findById(user).select('tenantId').lean()
|
||||
: null;
|
||||
if (owner)
|
||||
await closeResponses([
|
||||
{ conversationId: filter.conversationId, tenantId: owner.tenantId },
|
||||
]);
|
||||
/** A prior attempt may have deleted the root before a descendant read failed.
|
||||
* Resume from immutable root lineage and retain the root id for message,
|
||||
* checkpoint, and tool cleanup. The message probe distinguishes that partial
|
||||
* commit from a conversation id that never existed. */
|
||||
const [descendants, rootMessages] = await Promise.all([
|
||||
const [descendants, rootMessages, responseOwners] = await Promise.all([
|
||||
retryCascadeOperation(() =>
|
||||
Conversation.find({
|
||||
user,
|
||||
|
|
@ -4061,8 +4141,20 @@ export function createConversationMethods(
|
|||
.lean<DeletionConversation[]>(),
|
||||
),
|
||||
getMessages({ user, conversationId: filter.conversationId }, '_id', { limit: 1 }),
|
||||
discoverResponseOwners(mongoose, user, filter.conversationId),
|
||||
]);
|
||||
if (descendants.length === 0 && rootMessages.length === 0 && options?.allowEmpty !== true) {
|
||||
for (const responseOwner of responseOwners) {
|
||||
recoveryResponseScopes.push({
|
||||
conversationId: filter.conversationId,
|
||||
tenantId: responseOwner.tenantId,
|
||||
});
|
||||
}
|
||||
if (
|
||||
descendants.length === 0 &&
|
||||
rootMessages.length === 0 &&
|
||||
recoveryResponseScopes.length === 0 &&
|
||||
options?.allowEmpty !== true
|
||||
) {
|
||||
throw new Error('Conversation not found or already deleted.');
|
||||
}
|
||||
conversations = descendants;
|
||||
|
|
@ -4132,6 +4224,7 @@ export function createConversationMethods(
|
|||
break;
|
||||
}
|
||||
const waveIds = wave.map((conversation) => conversation.conversationId);
|
||||
await closeResponses(wave);
|
||||
await deps?.deleteAgentQueuedTurns?.(
|
||||
user,
|
||||
wave.map((conversation) => ({
|
||||
|
|
@ -4147,6 +4240,8 @@ export function createConversationMethods(
|
|||
conversationId: { $in: waveIds },
|
||||
}),
|
||||
);
|
||||
// Like subagent content, response erasure is an irreversible part of deletion intent.
|
||||
await eraseResponses(wave);
|
||||
const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } });
|
||||
if (result.deletedCount > 0) {
|
||||
/** Result erasure is irreversible. Keep receipts intact when a
|
||||
|
|
@ -4181,6 +4276,12 @@ export function createConversationMethods(
|
|||
];
|
||||
|
||||
if (recoveryConversationIds.length > 0) {
|
||||
// Each command retains an explicit owner/partition predicate. Request tenant
|
||||
// middleware must not replace the partitions discovered by recovery.
|
||||
await runAsSystem(async () => {
|
||||
await closeResponses(recoveryResponseScopes);
|
||||
await eraseResponses(recoveryResponseScopes);
|
||||
});
|
||||
await deps?.deleteAgentQueuedTurns?.(
|
||||
user,
|
||||
recoveryConversationIds.map((conversationId) => ({
|
||||
|
|
|
|||
|
|
@ -69,6 +69,20 @@ describe('conversation import cleanup methods', () => {
|
|||
text: record.conversationId,
|
||||
})),
|
||||
);
|
||||
await mongoose.models.StoredResponse.insertMany(
|
||||
records.map((record, index) => ({
|
||||
userId: record.user,
|
||||
tenantId: record.tenantId,
|
||||
conversationId: record.conversationId,
|
||||
responseId: `response-${index}`,
|
||||
requestKey: `request-${index}`,
|
||||
fingerprint: 'request',
|
||||
agentId: 'agent',
|
||||
requestJSON: '{"input":"private"}',
|
||||
parentMessageId: 'root',
|
||||
status: 'queued',
|
||||
})),
|
||||
);
|
||||
await mongoose.models.ChatProject.updateOne(
|
||||
{ _id: project._id },
|
||||
{
|
||||
|
|
@ -99,6 +113,9 @@ describe('conversation import cleanup methods', () => {
|
|||
});
|
||||
expect(remaining.conversations).toHaveLength(3);
|
||||
expect(remaining.messages).toHaveLength(3);
|
||||
expect(await runAsSystem(async () => mongoose.models.StoredResponse.countDocuments({}))).toBe(
|
||||
3,
|
||||
);
|
||||
expect(
|
||||
remaining.conversations.some(
|
||||
(record) =>
|
||||
|
|
@ -158,10 +175,17 @@ describe('conversation import cleanup methods', () => {
|
|||
const deleteMessages = jest.fn().mockResolvedValue(undefined);
|
||||
const findProjects = jest.fn().mockResolvedValue([]);
|
||||
const deleteConversations = jest.fn().mockResolvedValue(undefined);
|
||||
const responseScopes = { bulkWrite: jest.fn().mockResolvedValue(undefined) };
|
||||
const cleanupMongoose = {
|
||||
model: jest.fn().mockReturnValue(responseScopes),
|
||||
models: {
|
||||
Message: { deleteMany: deleteMessages },
|
||||
Conversation: { distinct: findProjects, deleteMany: deleteConversations },
|
||||
ResponseScope: responseScopes,
|
||||
StoredResponse: {
|
||||
updateMany: jest.fn().mockResolvedValue(undefined),
|
||||
deleteMany: jest.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
},
|
||||
} as unknown as typeof mongoose;
|
||||
const methods = createConversationImportMethods(cleanupMongoose);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { FilterQuery, Model } from 'mongoose';
|
||||
import type { IConversation, IMessage } from '~/types';
|
||||
import { closeResponseScopes, eraseResponseRecords } from './responseScope';
|
||||
import { runAsSystem, tenantStorage } from '~/config/tenantContext';
|
||||
import { refreshChatProjectStatsForUser } from './chatProject';
|
||||
|
||||
|
|
@ -77,6 +78,9 @@ export function createConversationImportMethods(
|
|||
affectedProjectIds.add(String(projectId));
|
||||
}
|
||||
}
|
||||
const owner = { userId: scope.user, tenantId: scope.tenantId };
|
||||
await closeResponseScopes(mongoose, owner, conversationIds);
|
||||
await eraseResponseRecords(mongoose, owner, conversationIds);
|
||||
await Conversation.deleteMany(filter);
|
||||
await mongoose.models.SubagentContent?.deleteMany(filter);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ import {
|
|||
import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCategory';
|
||||
/* Agent API Keys */
|
||||
import { createAgentApiKeyMethods, type AgentApiKeyMethods } from './agentApiKey';
|
||||
|
||||
/* MCP Servers */
|
||||
import { createMCPServerMethods, type MCPServerMethods } from './mcpServer';
|
||||
import { createCodeEnvironmentMethods, type CodeEnvironmentMethods } from './codeEnvironment';
|
||||
|
|
|
|||
1519
packages/data-schemas/src/methods/response.spec.ts
Normal file
1519
packages/data-schemas/src/methods/response.spec.ts
Normal file
File diff suppressed because it is too large
Load diff
691
packages/data-schemas/src/methods/response.ts
Normal file
691
packages/data-schemas/src/methods/response.ts
Normal file
|
|
@ -0,0 +1,691 @@
|
|||
import type {
|
||||
ResponseOwner,
|
||||
StoredResponseRecord,
|
||||
StoredResponseStatus,
|
||||
ResponseReviewCheckpoint,
|
||||
ResponseResumeIntent,
|
||||
ResponseTerminalReceipt,
|
||||
} from '~/models/response';
|
||||
import {
|
||||
responseScopeKeys,
|
||||
responseOwnerFilter,
|
||||
isResponseScopeClosed,
|
||||
initializeResponseScopes,
|
||||
} from './responseScope';
|
||||
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
|
||||
import { createResponseScopeModel } from '~/models/responseScope';
|
||||
import { createStoredResponseModel } from '~/models/response';
|
||||
import { getTransactionSupport } from '~/utils/transactions';
|
||||
import { activeExpirationFilter } from '~/utils/retention';
|
||||
import { createIndexesWithRetry } from '~/utils/retry';
|
||||
|
||||
export type {
|
||||
ResponseOwner,
|
||||
StoredResponseRecord,
|
||||
StoredResponseStatus,
|
||||
ResponseReviewCheckpoint,
|
||||
ResponseResumeIntent,
|
||||
ResponseTerminalReceipt,
|
||||
} from '~/models/response';
|
||||
|
||||
export interface ResponseReservation extends ResponseOwner {
|
||||
expiredAt?: Date;
|
||||
isTemporary?: boolean;
|
||||
responseId: string;
|
||||
requestKey: string;
|
||||
fingerprint: string;
|
||||
agentId: string;
|
||||
sourceKeyId?: string;
|
||||
requestJSON: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
}
|
||||
|
||||
export type ResponseReservationResult =
|
||||
| { ok: true; record: StoredResponseRecord; replayed: boolean }
|
||||
| { ok: false; code: 'idempotency_conflict' | 'response_unavailable' };
|
||||
|
||||
export type ResponseMutationResult =
|
||||
| { ok: true; record: StoredResponseRecord }
|
||||
| { ok: false; code: 'response_conflict' };
|
||||
|
||||
/** Internal drain identity only; expired requests, answers and decisions are never returned. */
|
||||
export type ResponseCleanupIdentity = Pick<
|
||||
StoredResponseRecord,
|
||||
| 'userId'
|
||||
| 'tenantId'
|
||||
| 'responseId'
|
||||
| 'agentId'
|
||||
| 'conversationId'
|
||||
| 'generationCreatedAt'
|
||||
| 'expiredAt'
|
||||
> & {
|
||||
/** Cleanup authority only; never permission to read output or execute a provider. */
|
||||
retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired';
|
||||
checkpointNamespace?: string;
|
||||
checkpointConfigJSON?: string;
|
||||
requiresCheckpointCleanup?: boolean;
|
||||
};
|
||||
|
||||
export interface ResponseResumeReservation
|
||||
extends Omit<ResponseResumeIntent, 'state' | 'reservedAt' | 'admittedAt'> {
|
||||
generationCreatedAt: number;
|
||||
}
|
||||
|
||||
/** Replays are receipts, not permission to enqueue or execute terminal/cancelled runs. */
|
||||
export type ResponseResumeResult =
|
||||
| { ok: true; record: StoredResponseRecord; replayed: boolean }
|
||||
| { ok: false; code: 'resume_conflict' };
|
||||
|
||||
export interface ResponseStorage {
|
||||
stageTerminal(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
revision: number,
|
||||
receipt: ResponseTerminalReceipt,
|
||||
): Promise<ResponseMutationResult>;
|
||||
finishTerminal(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
revision: number,
|
||||
receipt: ResponseTerminalReceipt,
|
||||
): Promise<ResponseMutationResult>;
|
||||
publishReview(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
revision: number,
|
||||
generationCreatedAt: number,
|
||||
review: ResponseReviewCheckpoint,
|
||||
responseJSON: string,
|
||||
): Promise<ResponseMutationResult>;
|
||||
reserveResume(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
input: ResponseResumeReservation,
|
||||
now: number,
|
||||
): Promise<ResponseResumeResult>;
|
||||
admitResume(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
intentId: string,
|
||||
generationCreatedAt: number,
|
||||
providerExecutionId: string,
|
||||
responseJSON: string,
|
||||
now: number,
|
||||
): Promise<ResponseMutationResult>;
|
||||
reserve(input: ResponseReservation): Promise<ResponseReservationResult>;
|
||||
read(owner: ResponseOwner, responseId: string): Promise<StoredResponseRecord | null>;
|
||||
readCleanupIdentity(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
retained?: ResponseCleanupIdentity,
|
||||
): Promise<ResponseCleanupIdentity | null>;
|
||||
attachDelivery(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
deliveryKey: string,
|
||||
): Promise<ResponseMutationResult>;
|
||||
publish(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
revision: number,
|
||||
update: { status: StoredResponseStatus; responseJSON: string; generationCreatedAt?: number },
|
||||
): Promise<ResponseMutationResult>;
|
||||
requestCancellation(
|
||||
owner: ResponseOwner,
|
||||
responseId: string,
|
||||
now: number,
|
||||
): Promise<StoredResponseRecord | null>;
|
||||
}
|
||||
|
||||
const active = ['queued', 'in_progress', 'requires_action'];
|
||||
|
||||
function sameTerminal(a: ResponseTerminalReceipt, b: ResponseTerminalReceipt): boolean {
|
||||
return (
|
||||
a.generationCreatedAt === b.generationCreatedAt &&
|
||||
a.providerExecutionId === b.providerExecutionId &&
|
||||
a.ownerId === b.ownerId &&
|
||||
a.status === b.status &&
|
||||
a.responseJSON === b.responseJSON &&
|
||||
a.checkpointNamespace === b.checkpointNamespace &&
|
||||
a.checkpointConfigJSON === b.checkpointConfigJSON
|
||||
);
|
||||
}
|
||||
|
||||
/** Authorization precedes this storage boundary; every read/write remains owner- and tenant-scoped. */
|
||||
export function createResponseStorage(mongoose: typeof import('mongoose')): ResponseStorage {
|
||||
const model = createStoredResponseModel(mongoose);
|
||||
let indexPromise: Promise<void> | undefined;
|
||||
let transactionSupport: boolean | null = null;
|
||||
function ensureIndexes() {
|
||||
indexPromise ??= createIndexesWithRetry(model)
|
||||
.then(() => undefined)
|
||||
.catch((error: unknown) => {
|
||||
indexPromise = undefined;
|
||||
throw error;
|
||||
});
|
||||
return indexPromise;
|
||||
}
|
||||
function ownerScope(owner: ResponseOwner) {
|
||||
if (!owner.userId || owner.tenantId === '__SYSTEM__')
|
||||
throw new TypeError('Invalid response owner');
|
||||
const tenantId = getTenantId() || undefined;
|
||||
if (
|
||||
tenantId != null &&
|
||||
tenantId !== SYSTEM_TENANT_ID &&
|
||||
tenantId !== (owner.tenantId || undefined)
|
||||
)
|
||||
throw new TypeError('Response tenant does not match authenticated scope');
|
||||
return responseOwnerFilter(owner);
|
||||
}
|
||||
const scope = (owner: ResponseOwner) => ({
|
||||
...ownerScope(owner),
|
||||
$and: [activeExpirationFilter<StoredResponseRecord>()],
|
||||
});
|
||||
return {
|
||||
async stageTerminal(owner, responseId, revision, receipt) {
|
||||
const pendingTerminal: ResponseTerminalReceipt = {
|
||||
generationCreatedAt: receipt.generationCreatedAt,
|
||||
providerExecutionId: receipt.providerExecutionId,
|
||||
...(receipt.ownerId == null ? {} : { ownerId: receipt.ownerId }),
|
||||
status: receipt.status,
|
||||
responseJSON: receipt.responseJSON,
|
||||
...(receipt.checkpointNamespace == null
|
||||
? {}
|
||||
: { checkpointNamespace: receipt.checkpointNamespace }),
|
||||
...(receipt.checkpointConfigJSON == null
|
||||
? {}
|
||||
: { checkpointConfigJSON: receipt.checkpointConfigJSON }),
|
||||
};
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
revision,
|
||||
generationCreatedAt: receipt.generationCreatedAt,
|
||||
status: { $in: active },
|
||||
pendingTerminal: { $exists: false },
|
||||
$or: [
|
||||
{
|
||||
'resume.state': 'admitted',
|
||||
'resume.providerExecutionId': receipt.providerExecutionId,
|
||||
},
|
||||
{
|
||||
'resume.state': { $ne: 'admitted' },
|
||||
'review.providerExecutionId': receipt.providerExecutionId,
|
||||
},
|
||||
{ review: { $exists: false }, resume: { $exists: false } },
|
||||
],
|
||||
},
|
||||
{ $set: { pendingTerminal }, $inc: { revision: 1 } },
|
||||
{ new: true, runValidators: true },
|
||||
)
|
||||
.lean();
|
||||
if (record) return { ok: true, record };
|
||||
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
|
||||
if (
|
||||
existing?.generationCreatedAt === receipt.generationCreatedAt &&
|
||||
existing.pendingTerminal &&
|
||||
sameTerminal(existing.pendingTerminal, pendingTerminal)
|
||||
)
|
||||
return { ok: true, record: existing };
|
||||
return { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async finishTerminal(owner, responseId, revision, receipt) {
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
revision,
|
||||
generationCreatedAt: receipt.generationCreatedAt,
|
||||
status: { $in: active },
|
||||
'pendingTerminal.generationCreatedAt': receipt.generationCreatedAt,
|
||||
'pendingTerminal.providerExecutionId': receipt.providerExecutionId,
|
||||
'pendingTerminal.ownerId': receipt.ownerId ?? { $exists: false },
|
||||
'pendingTerminal.status': receipt.status,
|
||||
'pendingTerminal.responseJSON': receipt.responseJSON,
|
||||
'pendingTerminal.checkpointNamespace': receipt.checkpointNamespace ?? {
|
||||
$exists: false,
|
||||
},
|
||||
'pendingTerminal.checkpointConfigJSON': receipt.checkpointConfigJSON ?? {
|
||||
$exists: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
$set: { status: receipt.status, responseJSON: receipt.responseJSON },
|
||||
$unset: { pendingTerminal: 1 },
|
||||
$inc: { revision: 1 },
|
||||
},
|
||||
{ new: true },
|
||||
)
|
||||
.lean();
|
||||
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async publishReview(owner, responseId, revision, generationCreatedAt, review, responseJSON) {
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
revision,
|
||||
generationCreatedAt,
|
||||
status: { $in: ['in_progress', 'requires_action'] },
|
||||
pendingTerminal: { $exists: false },
|
||||
cancellationRequestedAt: { $exists: false },
|
||||
$or: [
|
||||
{ review: { $exists: false }, resume: { $exists: false } },
|
||||
{
|
||||
'review.actionId': review.actionId,
|
||||
'review.providerExecutionId': review.providerExecutionId,
|
||||
resume: { $exists: false },
|
||||
},
|
||||
{
|
||||
'resume.state': 'admitted',
|
||||
'resume.providerExecutionId': review.providerExecutionId,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
status: 'requires_action',
|
||||
review: {
|
||||
actionId: review.actionId,
|
||||
providerExecutionId: review.providerExecutionId,
|
||||
expiresAt: review.expiresAt,
|
||||
partialResponseJSON: review.partialResponseJSON,
|
||||
...(review.visibleOutputTokens == null
|
||||
? {}
|
||||
: { visibleOutputTokens: review.visibleOutputTokens }),
|
||||
...(review.pendingActionJSON == null
|
||||
? {}
|
||||
: { pendingActionJSON: review.pendingActionJSON }),
|
||||
...(review.checkpointConfigJSON == null
|
||||
? {}
|
||||
: { checkpointConfigJSON: review.checkpointConfigJSON }),
|
||||
...(review.checkpointId == null ? {} : { checkpointId: review.checkpointId }),
|
||||
...(review.checkpointNs == null ? {} : { checkpointNs: review.checkpointNs }),
|
||||
},
|
||||
responseJSON,
|
||||
},
|
||||
$unset: { resume: 1 },
|
||||
$inc: { revision: 1 },
|
||||
},
|
||||
{ new: true, runValidators: true },
|
||||
)
|
||||
.lean();
|
||||
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async reserveResume(owner, responseId, input, now) {
|
||||
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume reservation time');
|
||||
if (input.providerExecutionId === input.previousProviderExecutionId)
|
||||
throw new TypeError('Resume requires a new provider execution identity');
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
generationCreatedAt: input.generationCreatedAt,
|
||||
status: 'requires_action',
|
||||
pendingTerminal: { $exists: false },
|
||||
cancellationRequestedAt: { $exists: false },
|
||||
resume: { $exists: false },
|
||||
'review.actionId': input.actionId,
|
||||
'review.providerExecutionId': input.previousProviderExecutionId,
|
||||
'review.expiresAt': { $gt: now },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
resume: {
|
||||
intentId: input.intentId,
|
||||
actionId: input.actionId,
|
||||
previousProviderExecutionId: input.previousProviderExecutionId,
|
||||
providerExecutionId: input.providerExecutionId,
|
||||
fingerprint: input.fingerprint,
|
||||
resolutionJSON: input.resolutionJSON,
|
||||
...(input.decisionJSON == null ? {} : { decisionJSON: input.decisionJSON }),
|
||||
state: 'reserved',
|
||||
reservedAt: now,
|
||||
},
|
||||
},
|
||||
$inc: { revision: 1 },
|
||||
},
|
||||
{ new: true, runValidators: true },
|
||||
)
|
||||
.lean();
|
||||
if (record) return { ok: true, record, replayed: false };
|
||||
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
|
||||
const intent = existing?.resume;
|
||||
if (
|
||||
existing?.generationCreatedAt === input.generationCreatedAt &&
|
||||
intent &&
|
||||
intent.actionId === input.actionId &&
|
||||
intent.previousProviderExecutionId === input.previousProviderExecutionId &&
|
||||
intent.fingerprint === input.fingerprint &&
|
||||
intent.resolutionJSON === input.resolutionJSON &&
|
||||
intent.decisionJSON === input.decisionJSON
|
||||
)
|
||||
return { ok: true, record: existing, replayed: true };
|
||||
return { ok: false, code: 'resume_conflict' };
|
||||
},
|
||||
async admitResume(
|
||||
owner,
|
||||
responseId,
|
||||
intentId,
|
||||
generationCreatedAt,
|
||||
providerExecutionId,
|
||||
responseJSON,
|
||||
now,
|
||||
) {
|
||||
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume admission time');
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
generationCreatedAt,
|
||||
status: 'requires_action',
|
||||
pendingTerminal: { $exists: false },
|
||||
cancellationRequestedAt: { $exists: false },
|
||||
'resume.intentId': intentId,
|
||||
'resume.providerExecutionId': providerExecutionId,
|
||||
'resume.state': 'reserved',
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
status: 'in_progress',
|
||||
responseJSON,
|
||||
'resume.state': 'admitted',
|
||||
'resume.admittedAt': now,
|
||||
},
|
||||
$inc: { revision: 1 },
|
||||
},
|
||||
{ new: true },
|
||||
)
|
||||
.lean();
|
||||
if (record) return { ok: true, record };
|
||||
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
|
||||
if (
|
||||
existing?.generationCreatedAt === generationCreatedAt &&
|
||||
existing.resume?.intentId === intentId &&
|
||||
existing.resume.providerExecutionId === providerExecutionId &&
|
||||
existing.resume.state === 'admitted'
|
||||
)
|
||||
return { ok: true, record: existing };
|
||||
return { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async reserve(input) {
|
||||
if (
|
||||
input.expiredAt != null &&
|
||||
(!(input.expiredAt instanceof Date) || !Number.isFinite(input.expiredAt.getTime()))
|
||||
)
|
||||
throw new TypeError('Invalid response expiration');
|
||||
const filter = { ...scope(input), requestKey: input.requestKey };
|
||||
transactionSupport = await getTransactionSupport(mongoose, transactionSupport);
|
||||
if (!transactionSupport) return { ok: false, code: 'response_unavailable' };
|
||||
await ensureIndexes();
|
||||
const prior = await model.findOne(filter).select('conversationId').lean();
|
||||
const keys = responseScopeKeys(input, prior?.conversationId ?? input.conversationId);
|
||||
await initializeResponseScopes(mongoose, keys);
|
||||
const session = await mongoose.startSession();
|
||||
try {
|
||||
let result: ResponseReservationResult | undefined;
|
||||
await session.withTransaction(async () => {
|
||||
// A write, not a snapshot read: erasure waits for admitted insertions to commit.
|
||||
const scopes = createResponseScopeModel(mongoose);
|
||||
const touched = await scopes.updateOne(
|
||||
{ _id: keys[0], closed: false },
|
||||
{ $inc: { revision: 1 } },
|
||||
{ session },
|
||||
);
|
||||
if (touched.matchedCount !== 1) {
|
||||
result = { ok: false, code: 'response_unavailable' };
|
||||
return;
|
||||
}
|
||||
const existing = await model.findOne(filter).session(session).lean();
|
||||
const conversationKey = responseScopeKeys(
|
||||
input,
|
||||
existing?.conversationId ?? input.conversationId,
|
||||
)[1];
|
||||
const conversation = await scopes
|
||||
.findOneAndUpdate(
|
||||
{ _id: conversationKey, closed: false },
|
||||
{
|
||||
$inc: { revision: 1 },
|
||||
...(input.expiredAt ? { $min: { expiredAt: input.expiredAt } } : {}),
|
||||
},
|
||||
{ session, new: true },
|
||||
)
|
||||
.lean();
|
||||
if (!conversation) {
|
||||
result = { ok: false, code: 'response_unavailable' };
|
||||
return;
|
||||
}
|
||||
if (conversation.expiredAt) {
|
||||
await model.updateMany(
|
||||
{
|
||||
...responseOwnerFilter(input),
|
||||
conversationId: existing?.conversationId ?? input.conversationId,
|
||||
$or: [{ expiredAt: null }, { expiredAt: { $gt: conversation.expiredAt } }],
|
||||
},
|
||||
{ $set: { expiredAt: conversation.expiredAt } },
|
||||
{ session },
|
||||
);
|
||||
if (existing && (!existing.expiredAt || conversation.expiredAt < existing.expiredAt))
|
||||
existing.expiredAt = conversation.expiredAt;
|
||||
if (conversation.expiredAt.getTime() <= Date.now()) {
|
||||
result = { ok: false, code: 'response_unavailable' };
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (existing) {
|
||||
result =
|
||||
existing.fingerprint === input.fingerprint
|
||||
? { ok: true, record: existing, replayed: true }
|
||||
: { ok: false, code: 'idempotency_conflict' };
|
||||
return;
|
||||
}
|
||||
const [created] = await model.create(
|
||||
[
|
||||
{
|
||||
...input,
|
||||
...(conversation.expiredAt ? { expiredAt: conversation.expiredAt } : {}),
|
||||
status: 'queued',
|
||||
revision: 0,
|
||||
},
|
||||
],
|
||||
{
|
||||
session,
|
||||
},
|
||||
);
|
||||
result = { ok: true, record: created.toObject(), replayed: false };
|
||||
});
|
||||
if (!result) throw new Error('Response reservation did not commit');
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 11000)
|
||||
throw error;
|
||||
const existing = await model.findOne(filter).lean();
|
||||
if (!existing) {
|
||||
if (await model.exists({ ...responseOwnerFilter(input), requestKey: input.requestKey }))
|
||||
return { ok: false, code: 'response_unavailable' };
|
||||
throw error;
|
||||
}
|
||||
if (await isResponseScopeClosed(mongoose, input, existing.conversationId))
|
||||
return { ok: false, code: 'response_unavailable' };
|
||||
return existing.fingerprint === input.fingerprint
|
||||
? { ok: true, record: existing, replayed: true }
|
||||
: { ok: false, code: 'idempotency_conflict' };
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
},
|
||||
async read(owner, responseId) {
|
||||
return model.findOne({ ...scope(owner), responseId }).lean();
|
||||
},
|
||||
async readCleanupIdentity(owner, responseId, retained) {
|
||||
const record = await model
|
||||
.findOne({ ...ownerScope(owner), responseId })
|
||||
.select(
|
||||
'userId tenantId responseId agentId conversationId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId',
|
||||
)
|
||||
.lean();
|
||||
// Retained data comes only from an internal job/receipt, never request input.
|
||||
// Validate it even when the row exists before using any fallback checkpoint fields.
|
||||
const matchingRetained =
|
||||
retained &&
|
||||
retained.responseId === responseId &&
|
||||
retained.userId === owner.userId &&
|
||||
(retained.tenantId || undefined) === (owner.tenantId || undefined) &&
|
||||
(!record ||
|
||||
(retained.conversationId === record.conversationId &&
|
||||
retained.agentId === record.agentId &&
|
||||
retained.generationCreatedAt === record.generationCreatedAt))
|
||||
? retained
|
||||
: undefined;
|
||||
const source = record ?? matchingRetained;
|
||||
if (!source) return null;
|
||||
const now = new Date();
|
||||
const [ownerKey, conversationKey] = responseScopeKeys(owner, source.conversationId);
|
||||
const anchors = await createResponseScopeModel(mongoose)
|
||||
.find({ _id: { $in: [ownerKey, conversationKey] } })
|
||||
.select('closed expiredAt')
|
||||
.lean();
|
||||
const ownerAnchor = anchors.find((anchor) => anchor._id === ownerKey);
|
||||
const conversationAnchor = anchors.find((anchor) => anchor._id === conversationKey);
|
||||
const expiredAt =
|
||||
conversationAnchor?.expiredAt &&
|
||||
(!source.expiredAt || conversationAnchor.expiredAt < source.expiredAt)
|
||||
? conversationAnchor.expiredAt
|
||||
: source.expiredAt;
|
||||
let retirement: ResponseCleanupIdentity['retirement'];
|
||||
if (ownerAnchor?.closed) retirement = 'owner_deleted';
|
||||
else if (conversationAnchor?.closed) retirement = 'conversation_deleted';
|
||||
else if (expiredAt && expiredAt <= now) retirement = 'expired';
|
||||
// A retained deadline alone cannot prove why an authoritative row disappeared.
|
||||
if (
|
||||
!record &&
|
||||
!ownerAnchor?.closed &&
|
||||
!conversationAnchor?.closed &&
|
||||
!(conversationAnchor?.expiredAt && conversationAnchor.expiredAt <= now)
|
||||
)
|
||||
return null;
|
||||
// A staged checkpoint is one binding: never fill an incomplete terminal
|
||||
// binding with settings from a prior review or retained provider segment.
|
||||
const terminalCheckpoint = record?.pendingTerminal;
|
||||
const hasTerminalCheckpoint =
|
||||
terminalCheckpoint?.checkpointNamespace != null ||
|
||||
terminalCheckpoint?.checkpointConfigJSON != null;
|
||||
const checkpointNamespace = hasTerminalCheckpoint
|
||||
? terminalCheckpoint?.checkpointNamespace
|
||||
: matchingRetained?.checkpointNamespace;
|
||||
const checkpointConfigJSON = hasTerminalCheckpoint
|
||||
? terminalCheckpoint?.checkpointConfigJSON
|
||||
: (record?.review?.checkpointConfigJSON ?? matchingRetained?.checkpointConfigJSON);
|
||||
const {
|
||||
userId,
|
||||
tenantId,
|
||||
responseId: id,
|
||||
agentId,
|
||||
conversationId,
|
||||
generationCreatedAt,
|
||||
} = source;
|
||||
return {
|
||||
userId,
|
||||
tenantId,
|
||||
responseId: id,
|
||||
agentId,
|
||||
conversationId,
|
||||
generationCreatedAt,
|
||||
expiredAt,
|
||||
retirement,
|
||||
checkpointNamespace,
|
||||
checkpointConfigJSON,
|
||||
requiresCheckpointCleanup: Boolean(
|
||||
record?.review ||
|
||||
checkpointNamespace ||
|
||||
checkpointConfigJSON ||
|
||||
matchingRetained?.requiresCheckpointCleanup,
|
||||
),
|
||||
};
|
||||
},
|
||||
async attachDelivery(owner, responseId, deliveryKey) {
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
status: { $in: active },
|
||||
$or: [{ deliveryKey: { $exists: false } }, { deliveryKey }],
|
||||
},
|
||||
{ $set: { deliveryKey } },
|
||||
{ new: true },
|
||||
)
|
||||
.lean();
|
||||
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async publish(owner, responseId, revision, update) {
|
||||
if (
|
||||
update.generationCreatedAt != null &&
|
||||
(!Number.isSafeInteger(update.generationCreatedAt) || update.generationCreatedAt <= 0)
|
||||
)
|
||||
throw new TypeError('Invalid response generation identity');
|
||||
let predecessors = active;
|
||||
if (update.status === 'queued') predecessors = ['queued'];
|
||||
if (update.status === 'in_progress') predecessors = ['queued', 'in_progress'];
|
||||
const record = await model
|
||||
.findOneAndUpdate(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
revision,
|
||||
status: { $in: predecessors },
|
||||
pendingTerminal: { $exists: false },
|
||||
...(update.status === 'in_progress'
|
||||
? {
|
||||
$and: [
|
||||
activeExpirationFilter<StoredResponseRecord>(),
|
||||
{
|
||||
$or: [
|
||||
{ status: 'in_progress' },
|
||||
{ cancellationRequestedAt: { $exists: false } },
|
||||
],
|
||||
},
|
||||
...(update.generationCreatedAt == null
|
||||
? [{ generationCreatedAt: { $gt: 0 } }]
|
||||
: []),
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
...(update.generationCreatedAt == null
|
||||
? {}
|
||||
: {
|
||||
$or: [
|
||||
{ generationCreatedAt: { $exists: false } },
|
||||
{ generationCreatedAt: update.generationCreatedAt },
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ $set: update, $inc: { revision: 1 } },
|
||||
{ new: true },
|
||||
)
|
||||
.lean();
|
||||
return record ? { ok: true, record } : { ok: false, code: 'response_conflict' };
|
||||
},
|
||||
async requestCancellation(owner, responseId, now) {
|
||||
if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid cancellation time');
|
||||
await model.updateOne(
|
||||
{
|
||||
...scope(owner),
|
||||
responseId,
|
||||
status: { $in: active },
|
||||
cancellationRequestedAt: { $exists: false },
|
||||
pendingTerminal: { $exists: false },
|
||||
},
|
||||
{ $set: { cancellationRequestedAt: now } },
|
||||
);
|
||||
return model.findOne({ ...scope(owner), responseId }).lean();
|
||||
},
|
||||
};
|
||||
}
|
||||
224
packages/data-schemas/src/methods/responseScope.ts
Normal file
224
packages/data-schemas/src/methods/responseScope.ts
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type { ResponseOwner } from '~/models/response';
|
||||
import { createResponseScopeModel } from '~/models/responseScope';
|
||||
import { createStoredResponseModel } from '~/models/response';
|
||||
import { getTransactionSupport } from '~/utils/transactions';
|
||||
import { createIndexesWithRetry } from '~/utils/retry';
|
||||
import { runAsSystem } from '~/config/tenantContext';
|
||||
|
||||
/** Response-bound conversation IDs are immutable: jobs and checkpoints retain that identity.
|
||||
* An empty source scope is closed before replacement so delayed admission cannot orphan data. */
|
||||
export function createResponseConversationReplacement(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): (owner: ResponseOwner, conversationId: string) => Promise<boolean> {
|
||||
let supported: boolean | null = null;
|
||||
return async (owner, conversationId) => {
|
||||
supported = await getTransactionSupport(mongoose, supported);
|
||||
const records = createStoredResponseModel(mongoose);
|
||||
const filter = { ...responseOwnerFilter(owner), conversationId };
|
||||
if (!supported) {
|
||||
// Reservation fails closed on standalone deployments, so no admission can race this check.
|
||||
if (await records.exists(filter)) return false;
|
||||
await closeResponseScopes(mongoose, owner, [conversationId]);
|
||||
return true;
|
||||
}
|
||||
const key = responseScopeKeys(owner, conversationId)[1];
|
||||
await initializeResponseScopes(mongoose, [key]);
|
||||
const session = await mongoose.startSession();
|
||||
try {
|
||||
let allowed = false;
|
||||
await session.withTransaction(async () => {
|
||||
const scopes = createResponseScopeModel(mongoose);
|
||||
// Serialize with the reservation's write to this same anchor before checking rows.
|
||||
await scopes.updateOne({ _id: key }, { $inc: { revision: 1 } }, { session });
|
||||
if (await records.exists(filter).session(session)) {
|
||||
allowed = false;
|
||||
return;
|
||||
}
|
||||
await scopes.updateOne({ _id: key }, { $set: { closed: true } }, { session });
|
||||
allowed = true;
|
||||
});
|
||||
return allowed;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Authorized missing-root recovery, restricted to one globally unique owner/conversation. */
|
||||
export async function discoverResponseOwners(
|
||||
mongoose: typeof import('mongoose'),
|
||||
userId: string,
|
||||
conversationId: string,
|
||||
): Promise<ResponseOwner[]> {
|
||||
return runAsSystem(async () => {
|
||||
const model = createStoredResponseModel(mongoose);
|
||||
const filter = { userId, conversationId };
|
||||
const [tenants, tenantless] = await Promise.all([
|
||||
model.distinct('tenantId', filter),
|
||||
model.exists({ ...filter, tenantId: { $in: [null, ''] } }),
|
||||
]);
|
||||
const owners: ResponseOwner[] = tenants
|
||||
.filter((tenantId): tenantId is string => typeof tenantId === 'string' && tenantId !== '')
|
||||
.map((tenantId) => ({ userId, tenantId }));
|
||||
if (tenantless) owners.push({ userId });
|
||||
return owners;
|
||||
});
|
||||
}
|
||||
|
||||
/** Retention intent is monotonic, like erasure: failed parent saves never reopen private data. */
|
||||
export function createResponseRetention(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): (owner: ResponseOwner, conversationId: string, expiredAt: Date) => Promise<void> {
|
||||
let supported: boolean | null = null;
|
||||
let indexes: Promise<void> | undefined;
|
||||
return async (owner, conversationId, expiredAt) => {
|
||||
if (!(expiredAt instanceof Date) || !Number.isFinite(expiredAt.getTime()))
|
||||
throw new TypeError('Invalid response expiration');
|
||||
const key = responseScopeKeys(owner, conversationId)[1];
|
||||
await initializeResponseScopes(mongoose, [key]);
|
||||
const scopes = createResponseScopeModel(mongoose);
|
||||
const records = createStoredResponseModel(mongoose);
|
||||
supported = await getTransactionSupport(mongoose, supported);
|
||||
if (supported) {
|
||||
indexes ??= createIndexesWithRetry(records)
|
||||
.then(() => undefined)
|
||||
.catch((error: unknown) => {
|
||||
indexes = undefined;
|
||||
throw error;
|
||||
});
|
||||
await indexes;
|
||||
}
|
||||
const apply = async (session?: import('mongoose').ClientSession) => {
|
||||
const anchor = await scopes
|
||||
.findOneAndUpdate(
|
||||
{ _id: key },
|
||||
{ $min: { expiredAt }, $inc: { revision: 1 } },
|
||||
{ new: true, ...(session ? { session } : {}) },
|
||||
)
|
||||
.lean();
|
||||
if (!anchor?.expiredAt) throw new Error('Response retention scope unavailable');
|
||||
await records.updateMany(
|
||||
{
|
||||
...responseOwnerFilter(owner),
|
||||
conversationId,
|
||||
$or: [{ expiredAt: null }, { expiredAt: { $gt: anchor.expiredAt } }],
|
||||
},
|
||||
{ $set: { expiredAt: anchor.expiredAt } },
|
||||
session ? { session } : {},
|
||||
);
|
||||
};
|
||||
if (!supported) return apply();
|
||||
const session = await mongoose.startSession();
|
||||
try {
|
||||
await session.withTransaction(() => apply(session));
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** The tenant policy treats missing, null and empty tenant IDs as one partition. */
|
||||
export function responseOwnerFilter(owner: ResponseOwner): {
|
||||
userId: string;
|
||||
tenantId: string | { $in: Array<null | string> };
|
||||
} {
|
||||
return { userId: owner.userId, tenantId: owner.tenantId || { $in: [null, ''] } };
|
||||
}
|
||||
|
||||
export async function eraseResponseRecords(
|
||||
mongoose: typeof import('mongoose'),
|
||||
owner: ResponseOwner,
|
||||
conversationIds?: readonly string[],
|
||||
): Promise<void> {
|
||||
await createStoredResponseModel(mongoose).deleteMany({
|
||||
...responseOwnerFilter(owner),
|
||||
...(conversationIds ? { conversationId: { $in: conversationIds } } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function scopeKey(owner: ResponseOwner, conversationId?: string): string {
|
||||
return createHash('sha256')
|
||||
.update(JSON.stringify([owner.tenantId || null, owner.userId, conversationId ?? null]))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
export function responseScopeKeys(owner: ResponseOwner, conversationId: string): string[] {
|
||||
return [scopeKey(owner), scopeKey(owner, conversationId)];
|
||||
}
|
||||
|
||||
/** Initializes only metadata; a late initializer can never reopen an erased scope. */
|
||||
export async function initializeResponseScopes(
|
||||
mongoose: typeof import('mongoose'),
|
||||
keys: string[],
|
||||
): Promise<void> {
|
||||
const model = createResponseScopeModel(mongoose);
|
||||
for (const key of keys) {
|
||||
try {
|
||||
await model.updateOne(
|
||||
{ _id: key },
|
||||
{ $setOnInsert: { closed: false, revision: 0 } },
|
||||
{ upsert: true },
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof mongoose.mongo.MongoServerError) ||
|
||||
error.code !== 11000 ||
|
||||
!(await model.exists({ _id: key }))
|
||||
)
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function isResponseScopeClosed(
|
||||
mongoose: typeof import('mongoose'),
|
||||
owner: ResponseOwner,
|
||||
conversationId: string,
|
||||
): Promise<boolean> {
|
||||
return (
|
||||
(await createResponseScopeModel(mongoose).exists({
|
||||
_id: { $in: responseScopeKeys(owner, conversationId) },
|
||||
closed: true,
|
||||
})) != null
|
||||
);
|
||||
}
|
||||
|
||||
/** Irreversible deletion intent. Ordinary retries complete erasure without readmitting work. */
|
||||
export async function closeResponseScopes(
|
||||
mongoose: typeof import('mongoose'),
|
||||
owner: ResponseOwner,
|
||||
conversationIds?: string[],
|
||||
): Promise<void> {
|
||||
const model = createResponseScopeModel(mongoose);
|
||||
const keys = conversationIds?.map((id) => scopeKey(owner, id)) ?? [scopeKey(owner)];
|
||||
// Bound each command independently of the number of conversations being removed.
|
||||
for (let offset = 0; offset < keys.length; offset += 250) {
|
||||
const operations = keys.slice(offset, offset + 250).map((key) => ({
|
||||
updateOne: {
|
||||
filter: { _id: key },
|
||||
update: { $set: { closed: true }, $setOnInsert: { revision: 0 } },
|
||||
upsert: true,
|
||||
},
|
||||
}));
|
||||
// Keys already hash tenant + owner; these payload-free anchors have no tenantId column.
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
const close = () => model.bulkWrite(operations, { ordered: false });
|
||||
try {
|
||||
await close();
|
||||
} catch (error) {
|
||||
if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 11000) throw error;
|
||||
await close();
|
||||
}
|
||||
}
|
||||
await createStoredResponseModel(mongoose).updateMany(
|
||||
{
|
||||
...responseOwnerFilter(owner),
|
||||
...(conversationIds ? { conversationId: { $in: conversationIds } } : {}),
|
||||
status: { $in: ['queued', 'in_progress', 'requires_action'] },
|
||||
cancellationRequestedAt: { $exists: false },
|
||||
pendingTerminal: { $exists: false },
|
||||
},
|
||||
{ $set: { cancellationRequestedAt: Date.now() } },
|
||||
);
|
||||
}
|
||||
30
packages/data-schemas/src/methods/test-helpers.spec.ts
Normal file
30
packages/data-schemas/src/methods/test-helpers.spec.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { waitForPause } from './test-helpers';
|
||||
|
||||
it('reaches a pause without waiting for the blocked operation', async () => {
|
||||
await expect(
|
||||
waitForPause(Promise.resolve(), new Promise<void>(() => undefined)),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects an operation that completes without reaching its pause', async () => {
|
||||
await expect(
|
||||
waitForPause(new Promise<void>(() => undefined), Promise.resolve('finished')),
|
||||
).rejects.toThrow('Operation completed before reaching its pause');
|
||||
});
|
||||
|
||||
it('preserves an operation failure before its pause', async () => {
|
||||
const failure = new Error('write failed');
|
||||
await expect(
|
||||
waitForPause(new Promise<void>(() => undefined), Promise.reject(failure)),
|
||||
).rejects.toBe(failure);
|
||||
});
|
||||
|
||||
it('observes a late operation failure after its pause was reached', async () => {
|
||||
let fail!: (reason: Error) => void;
|
||||
const operation = new Promise<void>((_resolve, reject) => {
|
||||
fail = reject;
|
||||
});
|
||||
await expect(waitForPause(Promise.resolve(), operation)).resolves.toBeUndefined();
|
||||
fail(new Error('late write failure'));
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
});
|
||||
|
|
@ -1,5 +1,15 @@
|
|||
import { gptPointReleaseFamily } from 'librechat-data-provider';
|
||||
|
||||
/** Fail promptly if an operation settles without reaching the expected pause. */
|
||||
export function waitForPause<T>(pause: Promise<void>, operation: Promise<T>): Promise<void> {
|
||||
return Promise.race([
|
||||
pause,
|
||||
operation.then(() => {
|
||||
throw new Error('Operation completed before reaching its pause');
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inlined utility functions previously imported from @librechat/api.
|
||||
* These are used only by test files in data-schemas.
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ import {
|
|||
import type * as t from '~/types';
|
||||
import { createToolApprovalGrantModel } from '~/models/toolApprovalGrant';
|
||||
import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';
|
||||
import { createStoredResponseModel } from '~/models/response';
|
||||
import { runAsSystem } from '~/config/tenantContext';
|
||||
import { createResponseStorage } from './response';
|
||||
import balanceSchema from '~/schema/balance';
|
||||
import userSchema from '~/schema/user';
|
||||
|
||||
|
|
@ -2362,3 +2365,85 @@ describe('personal grant cleanup before account deletion', () => {
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('private response cleanup before account deletion', () => {
|
||||
async function createResponses() {
|
||||
const user = await User.create({ email: 'delete-responses@example.com', provider: 'local' });
|
||||
const userId = user._id.toString();
|
||||
const model = createStoredResponseModel(mongoose);
|
||||
await runAsSystem(() =>
|
||||
model.create(
|
||||
[{ userId }, { userId, tenantId: 'another-tenant' }, { userId: 'another-owner' }].map(
|
||||
(scope, index) => ({
|
||||
...scope,
|
||||
responseId: `response-${index}`,
|
||||
requestKey: `request-${index}`,
|
||||
fingerprint: 'request',
|
||||
agentId: 'agent',
|
||||
requestJSON: '{"input":"private request"}',
|
||||
responseJSON: '{"output":"private reply"}',
|
||||
conversationId: 'conversation',
|
||||
parentMessageId: 'root',
|
||||
status: 'completed',
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
return { userId, model };
|
||||
}
|
||||
|
||||
it('removes private payloads before committing deletion and preserves other owners and tenants', async () => {
|
||||
const { userId, model } = await createResponses();
|
||||
const cleanup = jest.spyOn(model, 'deleteMany');
|
||||
const deletion = jest.spyOn(User, 'deleteOne');
|
||||
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
|
||||
expect(cleanup).toHaveBeenCalledWith({ userId, tenantId: { $in: [null, ''] } });
|
||||
expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(deletion.mock.invocationCallOrder[0]);
|
||||
await runAsSystem(async () => {
|
||||
expect(await model.countDocuments({ userId, tenantId: null })).toBe(0);
|
||||
expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(1);
|
||||
expect(await model.countDocuments({ userId: 'another-owner' })).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
it('erases every tenantless spelling when the legacy account stores an empty tenant', async () => {
|
||||
const { userId, model } = await createResponses();
|
||||
await User.updateOne({ _id: userId }, { $set: { tenantId: '' } });
|
||||
const original = (await model.findOne({ userId, responseId: 'response-0' }).lean())!;
|
||||
const { _id: _originalId, ...fields } = original;
|
||||
await runAsSystem(async () => {
|
||||
await model.create([
|
||||
{ ...fields, tenantId: '', responseId: 'empty-tenant', requestKey: 'empty-tenant' },
|
||||
{ ...fields, tenantId: null, responseId: 'null-tenant', requestKey: 'null-tenant' },
|
||||
]);
|
||||
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
|
||||
expect(await model.countDocuments({ userId, tenantId: { $in: [null, ''] } })).toBe(0);
|
||||
expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves the account on cleanup failure and allows a retry', async () => {
|
||||
const { userId, model } = await createResponses();
|
||||
jest
|
||||
.spyOn(model.collection, 'deleteMany')
|
||||
.mockRejectedValueOnce(new Error('response-store down'));
|
||||
const deletion = jest.spyOn(User, 'deleteOne');
|
||||
await expect(methods.deleteUserById(userId)).rejects.toThrow('response-store down');
|
||||
expect(deletion).not.toHaveBeenCalled();
|
||||
expect(await User.exists({ _id: userId })).not.toBeNull();
|
||||
expect(await createResponseStorage(mongoose).read({ userId }, 'response-0')).not.toBeNull();
|
||||
await methods.updateUser(userId, { name: 'Still usable' });
|
||||
expect((await User.findById(userId).lean())?.name).toBe('Still usable');
|
||||
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
|
||||
});
|
||||
|
||||
it('uses the account tenant when deletion runs under system authority', async () => {
|
||||
const { userId, model } = await createResponses();
|
||||
await User.updateOne({ _id: userId }, { $set: { tenantId: 'another-tenant' } });
|
||||
await runAsSystem(async () => {
|
||||
await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 });
|
||||
expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(0);
|
||||
expect(await model.countDocuments({ userId, tenantId: null })).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import type {
|
|||
} from '~/types';
|
||||
import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types';
|
||||
import type { CacheStore } from '~/types';
|
||||
import { closeResponseScopes, eraseResponseRecords } from './responseScope';
|
||||
import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search';
|
||||
import { evictAuthUserDocsForUsers } from '~/utils/eviction';
|
||||
import { signPayload } from '~/crypto';
|
||||
|
|
@ -641,6 +642,11 @@ export function createUserMethods(
|
|||
async function deleteUserById(userId: string): Promise<UserDeleteResult> {
|
||||
try {
|
||||
const User = mongoose.models.User;
|
||||
const owner = await User.findById(userId).select('tenantId').lean<Pick<IUser, 'tenantId'>>();
|
||||
if (owner) {
|
||||
await closeResponseScopes(mongoose, { userId, tenantId: owner.tenantId });
|
||||
await eraseResponseRecords(mongoose, { userId, tenantId: owner.tenantId });
|
||||
}
|
||||
await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId });
|
||||
const result = await User.deleteOne({ _id: userId });
|
||||
if (result.deletedCount === 0) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { createAgentQueuedTurnModel, createAgentQueuedTurnSequenceModel } from './queuedTurn';
|
||||
|
||||
import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';
|
||||
import { createScheduleModel, createScheduleRunModel } from './schedule';
|
||||
import { getTenantIndexMigrationHint } from '~/migrations/tenantIndexes';
|
||||
|
|
@ -10,12 +11,15 @@ import { createAgentTriggerDeliveryModel } from './triggerDelivery';
|
|||
import { createToolApprovalGrantModel } from './toolApprovalGrant';
|
||||
import { createSkillUploadIntentModel } from './skillUploadIntent';
|
||||
import { createSubagentContentModel } from './subagentContent';
|
||||
|
||||
import { createSkillSyncStatusModel } from './skillSyncStatus';
|
||||
import { createConversationTagModel } from './conversationTag';
|
||||
import { createCodeEnvironmentModel } from './codeEnvironment';
|
||||
import { createSkillDeletionModel } from './skillDeletion';
|
||||
import { createAgentCategoryModel } from './agentCategory';
|
||||
import { createResponseScopeModel } from './responseScope';
|
||||
import { createResourceLockModel } from './resourceLock';
|
||||
import { createStoredResponseModel } from './response';
|
||||
import { createChatProjectModel } from './chatProject';
|
||||
import { createAgentApiKeyModel } from './agentApiKey';
|
||||
import { createTransactionModel } from './transaction';
|
||||
|
|
@ -57,6 +61,8 @@ import logger from '~/config/winston';
|
|||
*/
|
||||
export function createModels(mongoose: typeof import('mongoose')): {
|
||||
User: ReturnType<typeof createUserModel>;
|
||||
StoredResponse: ReturnType<typeof createStoredResponseModel>;
|
||||
ResponseScope: ReturnType<typeof createResponseScopeModel>;
|
||||
ToolApprovalGrant: ReturnType<typeof createToolApprovalGrantModel>;
|
||||
Token: ReturnType<typeof createTokenModel>;
|
||||
Session: ReturnType<typeof createSessionModel>;
|
||||
|
|
@ -112,6 +118,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
|
|||
} {
|
||||
const models = {
|
||||
User: createUserModel(mongoose),
|
||||
StoredResponse: createStoredResponseModel(mongoose),
|
||||
ResponseScope: createResponseScopeModel(mongoose),
|
||||
ToolApprovalGrant: createToolApprovalGrantModel(mongoose),
|
||||
Token: createTokenModel(mongoose),
|
||||
Session: createSessionModel(mongoose),
|
||||
|
|
|
|||
195
packages/data-schemas/src/models/response.ts
Normal file
195
packages/data-schemas/src/models/response.ts
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import { Schema } from 'mongoose';
|
||||
import type { Model } from 'mongoose';
|
||||
import { applyTenantIsolation } from './plugins/tenantIsolation';
|
||||
|
||||
export interface ResponseOwner {
|
||||
userId: string;
|
||||
tenantId?: string;
|
||||
}
|
||||
|
||||
export type StoredResponseStatus =
|
||||
| 'queued'
|
||||
| 'in_progress'
|
||||
| 'requires_action'
|
||||
| 'completed'
|
||||
| 'failed'
|
||||
| 'incomplete'
|
||||
| 'cancelled';
|
||||
|
||||
export interface ResponseReviewCheckpoint {
|
||||
actionId: string;
|
||||
providerExecutionId: string;
|
||||
expiresAt: number;
|
||||
/** Private seed for the resumed output; never part of the public response view. */
|
||||
partialResponseJSON: string;
|
||||
/** Visible assistant token counter, separate from billed multi-model usage. */
|
||||
visibleOutputTokens?: number;
|
||||
/** Private reviewed graph/tool/checkpoint bindings retained after the job consumes its action. */
|
||||
pendingActionJSON?: string;
|
||||
/** Exact checkpointer settings captured by the paused run, independent of later config changes. */
|
||||
checkpointConfigJSON?: string;
|
||||
checkpointId?: string;
|
||||
checkpointNs?: string;
|
||||
}
|
||||
|
||||
export interface ResponseResumeIntent {
|
||||
intentId: string;
|
||||
actionId: string;
|
||||
previousProviderExecutionId: string;
|
||||
providerExecutionId: string;
|
||||
fingerprint: string;
|
||||
/** Validated, owner-submitted SDK resolution; not a credential or executable request. */
|
||||
resolutionJSON: string;
|
||||
/** Original validated browser decision, including edited arguments and remembered-approval scope. */
|
||||
decisionJSON?: string;
|
||||
state: 'reserved' | 'admitted';
|
||||
reservedAt: number;
|
||||
admittedAt?: number;
|
||||
}
|
||||
|
||||
/** Server-written receipt after provider and trailing writes have definitively settled. */
|
||||
export interface ResponseTerminalReceipt {
|
||||
generationCreatedAt: number;
|
||||
providerExecutionId: string;
|
||||
ownerId?: string;
|
||||
status: 'completed' | 'failed' | 'incomplete' | 'cancelled';
|
||||
responseJSON: string;
|
||||
checkpointNamespace?: string;
|
||||
/** Server-pinned saver settings, including initial failures before a review exists. */
|
||||
checkpointConfigJSON?: string;
|
||||
}
|
||||
|
||||
/** Durable identity and dispatch/result journal for a background Responses request. */
|
||||
export interface StoredResponseRecord {
|
||||
expiredAt?: Date;
|
||||
isTemporary?: boolean;
|
||||
tenantId?: string;
|
||||
userId: string;
|
||||
responseId: string;
|
||||
requestKey: string;
|
||||
fingerprint: string;
|
||||
agentId: string;
|
||||
sourceKeyId?: string;
|
||||
requestJSON: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
status: StoredResponseStatus;
|
||||
revision: number;
|
||||
deliveryKey?: string;
|
||||
generationCreatedAt?: number;
|
||||
responseJSON?: string;
|
||||
cancellationRequestedAt?: number;
|
||||
review?: ResponseReviewCheckpoint;
|
||||
resume?: ResponseResumeIntent;
|
||||
pendingTerminal?: ResponseTerminalReceipt;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export function createStoredResponseModel(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): Model<StoredResponseRecord> {
|
||||
const schema = new Schema<StoredResponseRecord>(
|
||||
{
|
||||
tenantId: String,
|
||||
expiredAt: Date,
|
||||
isTemporary: Boolean,
|
||||
userId: { type: String, required: true },
|
||||
responseId: { type: String, required: true },
|
||||
requestKey: { type: String, required: true },
|
||||
fingerprint: { type: String, required: true },
|
||||
agentId: { type: String, required: true },
|
||||
sourceKeyId: String,
|
||||
requestJSON: { type: String, required: true },
|
||||
conversationId: { type: String, required: true },
|
||||
parentMessageId: { type: String, required: true },
|
||||
status: {
|
||||
type: String,
|
||||
required: true,
|
||||
enum: [
|
||||
'queued',
|
||||
'in_progress',
|
||||
'requires_action',
|
||||
'completed',
|
||||
'failed',
|
||||
'incomplete',
|
||||
'cancelled',
|
||||
],
|
||||
},
|
||||
revision: { type: Number, required: true, default: 0 },
|
||||
deliveryKey: String,
|
||||
generationCreatedAt: Number,
|
||||
responseJSON: String,
|
||||
cancellationRequestedAt: Number,
|
||||
pendingTerminal: {
|
||||
type: new Schema<ResponseTerminalReceipt>(
|
||||
{
|
||||
generationCreatedAt: { type: Number, required: true, min: 1 },
|
||||
providerExecutionId: { type: String, required: true, maxlength: 128 },
|
||||
ownerId: { type: String, maxlength: 128 },
|
||||
status: {
|
||||
type: String,
|
||||
required: true,
|
||||
enum: ['completed', 'failed', 'incomplete', 'cancelled'],
|
||||
},
|
||||
responseJSON: { type: String, required: true },
|
||||
checkpointNamespace: String,
|
||||
checkpointConfigJSON: String,
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
required: false,
|
||||
},
|
||||
review: {
|
||||
type: new Schema<ResponseReviewCheckpoint>(
|
||||
{
|
||||
actionId: { type: String, required: true, maxlength: 256 },
|
||||
providerExecutionId: { type: String, required: true, maxlength: 128 },
|
||||
expiresAt: { type: Number, required: true, min: 1 },
|
||||
partialResponseJSON: { type: String, required: true },
|
||||
visibleOutputTokens: {
|
||||
type: Number,
|
||||
min: 0,
|
||||
max: Number.MAX_SAFE_INTEGER,
|
||||
validate: Number.isSafeInteger,
|
||||
},
|
||||
pendingActionJSON: String,
|
||||
checkpointConfigJSON: String,
|
||||
checkpointId: String,
|
||||
checkpointNs: String,
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
required: false,
|
||||
},
|
||||
resume: {
|
||||
type: new Schema<ResponseResumeIntent>(
|
||||
{
|
||||
intentId: { type: String, required: true, maxlength: 256 },
|
||||
actionId: { type: String, required: true, maxlength: 256 },
|
||||
previousProviderExecutionId: { type: String, required: true, maxlength: 128 },
|
||||
providerExecutionId: { type: String, required: true, maxlength: 128 },
|
||||
fingerprint: { type: String, required: true },
|
||||
resolutionJSON: { type: String, required: true },
|
||||
decisionJSON: String,
|
||||
state: { type: String, required: true, enum: ['reserved', 'admitted'] },
|
||||
reservedAt: { type: Number, required: true, min: 1 },
|
||||
admittedAt: { type: Number, min: 1 },
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
required: false,
|
||||
},
|
||||
},
|
||||
{ timestamps: true, minimize: false },
|
||||
);
|
||||
schema.index({ tenantId: 1, userId: 1, responseId: 1 }, { unique: true });
|
||||
schema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
|
||||
schema.index({ tenantId: 1, userId: 1, requestKey: 1 }, { unique: true });
|
||||
schema.index({ tenantId: 1, userId: 1, conversationId: 1, createdAt: 1 });
|
||||
schema.index({ userId: 1, conversationId: 1, tenantId: 1 });
|
||||
applyTenantIsolation(schema);
|
||||
return (
|
||||
mongoose.models.StoredResponse ?? mongoose.model<StoredResponseRecord>('StoredResponse', schema)
|
||||
);
|
||||
}
|
||||
26
packages/data-schemas/src/models/responseScope.ts
Normal file
26
packages/data-schemas/src/models/responseScope.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { Schema } from 'mongoose';
|
||||
import type { Model } from 'mongoose';
|
||||
|
||||
interface ResponseScope {
|
||||
_id: string;
|
||||
closed: boolean;
|
||||
revision: number;
|
||||
expiredAt?: Date;
|
||||
}
|
||||
|
||||
/** Payload-free admission anchors; closed scopes remain tombstones after erasure. */
|
||||
export function createResponseScopeModel(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): Model<ResponseScope> {
|
||||
if (mongoose.models.ResponseScope) return mongoose.model<ResponseScope>('ResponseScope');
|
||||
const schema = new Schema<ResponseScope>(
|
||||
{
|
||||
_id: { type: String, required: true },
|
||||
closed: { type: Boolean, required: true, default: false },
|
||||
revision: { type: Number, required: true, default: 0 },
|
||||
expiredAt: Date,
|
||||
},
|
||||
{ versionKey: false },
|
||||
);
|
||||
return mongoose.model<ResponseScope>('ResponseScope', schema);
|
||||
}
|
||||
|
|
@ -1,105 +1,76 @@
|
|||
import logger from '~/config/winston';
|
||||
|
||||
export const CANCEL_RATE = 1.15;
|
||||
|
||||
const TRANSACTION_PROBE_COLLECTION = '__transaction_test__';
|
||||
|
||||
function unsupportedTransaction(error: unknown, mongoose: typeof import('mongoose')): boolean {
|
||||
return (
|
||||
error instanceof mongoose.mongo.MongoServerError &&
|
||||
error.code === 20 &&
|
||||
/Transaction numbers are only allowed on a replica set member or mongos/i.test(error.message)
|
||||
);
|
||||
}
|
||||
|
||||
async function probeTransaction(mongoose: typeof import('mongoose')): Promise<boolean> {
|
||||
const session = await mongoose.startSession();
|
||||
try {
|
||||
const session = await mongoose.startSession();
|
||||
try {
|
||||
session.startTransaction();
|
||||
|
||||
await mongoose.connection.db
|
||||
?.collection(TRANSACTION_PROBE_COLLECTION)
|
||||
.findOne({}, { session });
|
||||
|
||||
await session.commitTransaction();
|
||||
logger.debug('MongoDB transactions are supported');
|
||||
return true;
|
||||
} catch (transactionError: unknown) {
|
||||
try {
|
||||
await session.abortTransaction();
|
||||
} catch (transactionError) {
|
||||
/** best-effort abort */
|
||||
logger.error(`[supportsTransactions] Error aborting transaction:`, transactionError);
|
||||
}
|
||||
logger.debug(
|
||||
'MongoDB transactions not supported (transaction error):',
|
||||
(transactionError as Error)?.message || 'Unknown error',
|
||||
);
|
||||
return false;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
session.startTransaction();
|
||||
const db = mongoose.connection.db;
|
||||
if (!db) throw new Error('Transaction capability probe requires a connected database');
|
||||
await db.collection(TRANSACTION_PROBE_COLLECTION).findOne({}, { session });
|
||||
await session.commitTransaction();
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.debug(
|
||||
'MongoDB transactions not supported (session error):',
|
||||
(error as Error)?.message || 'Unknown error',
|
||||
);
|
||||
return false;
|
||||
try {
|
||||
await session.abortTransaction();
|
||||
} catch {
|
||||
logger.debug('[supportsTransactions] Probe transaction abort failed');
|
||||
}
|
||||
if (unsupportedTransaction(error, mongoose)) return false;
|
||||
throw error;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the connected MongoDB deployment supports transactions
|
||||
* This requires a MongoDB replica set configuration
|
||||
*
|
||||
* Amazon DocumentDB rejects a transaction that touches a collection which does
|
||||
* not exist ("Feature not supported: non-existent collection in transaction"),
|
||||
* so a failed first probe materializes the canary and probes once more —
|
||||
* otherwise engines that fully support transactions would report unsupported
|
||||
* and silently drop every caller to the non-transactional path. MongoDB allows
|
||||
* transactional reads of missing collections, so deployments that pass the
|
||||
* first probe never pay the create and never gain the canary collection. A
|
||||
* failed create is logged rather than swallowed: on a hardened role it is the
|
||||
* only visible explanation for a wrong "unsupported" verdict.
|
||||
*
|
||||
* @returns True if transactions are supported, false otherwise
|
||||
*/
|
||||
/** Only confirmed standalone incompatibility is false; operational failures stay retryable. */
|
||||
export const supportsTransactions = async (
|
||||
mongoose: typeof import('mongoose'),
|
||||
): Promise<boolean> => {
|
||||
if (await probeTransaction(mongoose)) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
await mongoose.connection.db?.createCollection(TRANSACTION_PROBE_COLLECTION);
|
||||
return await probeTransaction(mongoose);
|
||||
} catch (error) {
|
||||
const message = (error as Error)?.message ?? '';
|
||||
if (!/already exists|NamespaceExists/i.test(message)) {
|
||||
logger.warn(
|
||||
`[supportsTransactions] Could not create the "${TRANSACTION_PROBE_COLLECTION}" probe collection; ` +
|
||||
'if this deployment is Amazon DocumentDB, the transaction-support verdict below may be a false negative:',
|
||||
error,
|
||||
);
|
||||
// DocumentDB requires the probe collection to exist before a transaction can read it.
|
||||
if (
|
||||
!(error instanceof mongoose.mongo.MongoServerError) ||
|
||||
!/non-existent collection in transaction/i.test(error.message)
|
||||
)
|
||||
throw error;
|
||||
const db = mongoose.connection.db;
|
||||
if (!db) throw error;
|
||||
try {
|
||||
await db.createCollection(TRANSACTION_PROBE_COLLECTION);
|
||||
} catch (creationError) {
|
||||
if (!(creationError instanceof mongoose.mongo.MongoServerError) || creationError.code !== 48)
|
||||
throw creationError;
|
||||
}
|
||||
return probeTransaction(mongoose);
|
||||
}
|
||||
return probeTransaction(mongoose);
|
||||
};
|
||||
|
||||
let inflightProbe: Promise<boolean> | null = null;
|
||||
const inflightProbes = new WeakMap<import('mongoose').Connection, Promise<boolean>>();
|
||||
|
||||
/**
|
||||
* Gets whether the current MongoDB deployment supports transactions
|
||||
* Caches the result for performance
|
||||
*
|
||||
* Concurrent first callers share one in-flight probe instead of each paying
|
||||
* the session/transaction round trips before the caller-side cache fills.
|
||||
*
|
||||
* @returns True if transactions are supported, false otherwise
|
||||
*/
|
||||
/** Share first probes per connection. Callers cache only a successful capability verdict. */
|
||||
export const getTransactionSupport = async (
|
||||
mongoose: typeof import('mongoose'),
|
||||
transactionSupportCache: boolean | null,
|
||||
): Promise<boolean> => {
|
||||
if (transactionSupportCache !== null) {
|
||||
return transactionSupportCache;
|
||||
if (transactionSupportCache !== null) return transactionSupportCache;
|
||||
const connection = mongoose.connection;
|
||||
let probe = inflightProbes.get(connection);
|
||||
if (!probe) {
|
||||
probe = supportsTransactions(mongoose).finally(() => inflightProbes.delete(connection));
|
||||
inflightProbes.set(connection, probe);
|
||||
}
|
||||
if (inflightProbe == null) {
|
||||
inflightProbe = supportsTransactions(mongoose).finally(() => {
|
||||
inflightProbe = null;
|
||||
});
|
||||
}
|
||||
return await inflightProbe;
|
||||
return probe;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue