diff --git a/.github/workflows/change-quality.yml b/.github/workflows/change-quality.yml index bdec5ee2bd..a1f3ef1c9a 100644 --- a/.github/workflows/change-quality.yml +++ b/.github/workflows/change-quality.yml @@ -74,7 +74,8 @@ jobs: needs: [select, verify] if: needs.select.outputs.hasChanges == 'true' runs-on: ubuntu-latest - timeout-minutes: 45 + # Whole edited database functions can produce thousands of mutation cases. + timeout-minutes: ${{ matrix.check == 'mutation' && 240 || 45 }} defaults: run: working-directory: source diff --git a/e2e/specs/mock/chat.spec.ts b/e2e/specs/mock/chat.spec.ts index 4374d7005e..a52a11fdd6 100644 --- a/e2e/specs/mock/chat.spec.ts +++ b/e2e/specs/mock/chat.spec.ts @@ -341,7 +341,9 @@ test.describe('core chat loop', () => { await uploadProviderFile(page, csvFixture); - await expect(page.getByRole('button', { name: filename })).toBeVisible(); + await expect( + page.getByTestId('composer-tray').getByRole('button', { name: filename, exact: true }), + ).toBeVisible(); const input = page.getByRole('textbox', { name: 'Message input' }); await input.click(); diff --git a/e2e/specs/mock/queued-reveal.spec.ts b/e2e/specs/mock/queued-reveal.spec.ts index b7fc736335..6682bd3478 100644 --- a/e2e/specs/mock/queued-reveal.spec.ts +++ b/e2e/specs/mock/queued-reveal.spec.ts @@ -1,4 +1,5 @@ import { expect, test } from '@playwright/test'; +import type { TMessage } from 'librechat-data-provider'; import type { Page, Route } from '@playwright/test'; import type { AgentDetail } from './agents.helpers'; import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers'; @@ -27,6 +28,7 @@ const messageTurns = (page: Page) => messagesView(page).locator('.message-render const RECEIPTS_ROUTE = /\/api\/agents\/chat\/queued-turns\?/; const STATUS_ROUTE = /\/api\/agents\/chat\/status\//; +const MESSAGES_ROUTE = /\/api\/messages\/[^/?]+(?:\?|$)/; type ReceiptsBody = { queuedTurns?: Array<{ status?: string }> }; type StatusBody = { active?: boolean; resumeState?: { userMessage?: { text?: string } } }; @@ -39,8 +41,9 @@ type StatusBody = { active?: boolean; resumeState?: { userMessage?: { text?: str * only through the receipt projection and the status it re-arms. While held * back, the receipts stay frozen at their pre-admission snapshot and a status * that describes the successor reads as inactive, so the next user turn can - * only come from the completion event itself. Every other request, including - * the predecessor's own terminal status, passes through untouched. + * only come from the completion event itself. History also withholds the admitted + * successor and its descendants, so a backfill cannot bypass the hold. Other requests, + * including the predecessor's own terminal status, pass through untouched. */ function holdBackSuccessor(page: Page, successorText: string) { let holding = false; @@ -70,10 +73,39 @@ function holdBackSuccessor(page: Page, successorText: string) { json: describesSuccessor ? { active: false } : body, }); }; + const messages = async (route: Route) => { + const response = await route.fetch(); + if (!holding || response.status() !== 200 || route.request().method() !== 'GET') { + return route.fulfill({ response }); + } + const body = (await response.json()) as TMessage[]; + const hidden = new Set(); + const children = new Map(); + for (const message of body) { + if (message.isCreatedByUser && message.text === successorText) hidden.add(message.messageId); + const parent = message.parentMessageId ?? ''; + const siblings = children.get(parent) ?? []; + siblings.push(message.messageId); + children.set(parent, siblings); + } + const pending = [...hidden]; + for (const id of pending) { + for (const child of children.get(id) ?? []) { + if (hidden.has(child)) continue; + hidden.add(child); + pending.push(child); + } + } + return route.fulfill({ + response, + json: body.filter((message) => !hidden.has(message.messageId)), + }); + }; return { arm: async () => { await page.route(RECEIPTS_ROUTE, receipts); await page.route(STATUS_ROUTE, status); + await page.route(MESSAGES_ROUTE, messages); }, hold: () => { holding = true; @@ -82,6 +114,7 @@ function holdBackSuccessor(page: Page, successorText: string) { holding = false; await page.unroute(RECEIPTS_ROUTE, receipts); await page.unroute(STATUS_ROUTE, status); + await page.unroute(MESSAGES_ROUTE, messages); }, }; } @@ -166,13 +199,14 @@ test.describe('server-queued follow-up reveal', () => { const predecessor = messageTurns(page).nth(3); await expect(predecessor).toContainText(SLOW_REPLY_LAST_CHUNK, { timeout: 60000 }); + const pending = page.getByTestId('pending-turn'); + await expect(pending).toBeVisible({ timeout: 5000 }); const queuedTurn = messageTurns(page).nth(4); await expect(queuedTurn).toContainText(queueText, { timeout: 5000 }); await expect(queuedTurn.locator('.user-turn')).toBeVisible(); /** One visible copy: actions follow the user turn, not a duplicate chip * below the composer. Other queued turns still use their ordinary rows. */ await expect(row).toHaveCount(0); - const pending = page.getByTestId('pending-turn'); await expect(pending).toContainText('Starting as the next turn'); await expect(pending.getByRole('button', { name: 'Send now' })).toHaveCount(0); await expect(pending.getByRole('button', { name: 'Remove message' })).toBeVisible(); diff --git a/e2e/specs/mock/scenarios/compaction-rerun-controls.spec.ts b/e2e/specs/mock/scenarios/compaction-rerun-controls.spec.ts index f17a678bec..5f93df46c8 100644 --- a/e2e/specs/mock/scenarios/compaction-rerun-controls.spec.ts +++ b/e2e/specs/mock/scenarios/compaction-rerun-controls.spec.ts @@ -507,7 +507,7 @@ test.describe('compaction rerun controls', () => { await page.locator(`[id="edit-${chainedId}"]`).click(); const editor = page.getByTestId('message-text-editor'); await expect(editor).toBeVisible(); - await expect(page.getByRole('button', { name: 'Save' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Save', exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Rerun', exact: true })).toHaveCount(0); await expect(page.getByRole('button', { name: 'Update & rerun' })).toHaveCount(0); /* ...and the footer says why the action it usually carries is gone. */ @@ -545,7 +545,7 @@ test.describe('compaction rerun controls', () => { await page.locator(`[id="edit-${rootReplyId}"]`).click(); const editor = page.getByTestId('message-text-editor'); await expect(editor).toBeVisible(); - await expect(page.getByRole('button', { name: 'Save' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Save', exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Rerun', exact: true })).toHaveCount(0); await expect( page.getByText('Rerunning replays the user message before a response', { exact: false }), diff --git a/e2e/specs/mock/scenarios/deleted-storage-attachments.spec.ts b/e2e/specs/mock/scenarios/deleted-storage-attachments.spec.ts index 52e9d1fa3f..cbe8964114 100644 --- a/e2e/specs/mock/scenarios/deleted-storage-attachments.spec.ts +++ b/e2e/specs/mock/scenarios/deleted-storage-attachments.spec.ts @@ -41,7 +41,9 @@ async function attachMarkdown(page: Page) { expect(response.ok()).toBeTruthy(); const file = (await response.json()) as UploadedFile & { filepath?: string }; expect(file.llmDeliveryPath).toBe('provider'); - await expect(page.getByRole('button', { name })).toBeVisible({ timeout: 15000 }); + await expect( + page.getByTestId('composer-tray').getByRole('button', { name, exact: true }), + ).toBeVisible({ timeout: 15000 }); return { name, file }; } diff --git a/e2e/specs/mock/scenarios/google-media-tokens.spec.ts b/e2e/specs/mock/scenarios/google-media-tokens.spec.ts index 00c7144278..33cedb6e32 100644 --- a/e2e/specs/mock/scenarios/google-media-tokens.spec.ts +++ b/e2e/specs/mock/scenarios/google-media-tokens.spec.ts @@ -87,7 +87,9 @@ test.describe('Google native media in the context budget', () => { }); expect(upload.ok()).toBeTruthy(); expect(((await upload.json()) as UploadedFile).llmDeliveryPath).toBe('provider'); - await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 }); + await expect( + page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }), + ).toBeVisible({ timeout: 15000 }); const report = await reportContextTokens(page); expect(report.provider).toBe('google'); @@ -107,7 +109,9 @@ test.describe('Google native media in the context budget', () => { content: syntheticPdf(), }); expect(upload.ok()).toBeTruthy(); - await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 }); + await expect( + page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }), + ).toBeVisible({ timeout: 15000 }); const first = await reportContextTokens(page); expect(first.latest).toBeGreaterThan(ONE_PAGE_TOKENS); diff --git a/e2e/specs/mock/shared-links.spec.ts b/e2e/specs/mock/shared-links.spec.ts index 0a040fc103..4b6d688cf0 100644 --- a/e2e/specs/mock/shared-links.spec.ts +++ b/e2e/specs/mock/shared-links.spec.ts @@ -148,7 +148,11 @@ test.describe('shared links', () => { await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); await uploadProviderFile(page, fileFixture); - await expect(page.getByRole('button', { name: fileFixture.name, exact: true })).toBeVisible(); + await expect( + page + .getByTestId('composer-tray') + .getByRole('button', { name: fileFixture.name, exact: true }), + ).toBeVisible(); const response = await sendMessage(page, userMessage); expect(response.ok()).toBeTruthy(); diff --git a/e2e/specs/mock/skill-file-authoring.spec.ts b/e2e/specs/mock/skill-file-authoring.spec.ts index 7614bf460f..575201cade 100644 --- a/e2e/specs/mock/skill-file-authoring.spec.ts +++ b/e2e/specs/mock/skill-file-authoring.spec.ts @@ -125,8 +125,8 @@ test.describe('skill file authoring tools', () => { await expect(page.getByRole('heading', { level: 1, name: skillName })).toBeVisible(); await expect(page.getByText(EDITED_DESCRIPTION)).toBeVisible(); - await page.getByRole('button', { name: 'Delete' }).click(); - await page.getByRole('dialog').getByRole('button', { name: 'Delete' }).click(); + await page.getByTestId('app-pane').getByRole('button', { name: 'Delete', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'Delete', exact: true }).click(); await expect(page).toHaveURL((url) => url.pathname === '/skills'); const token = await getAccessToken(page); await expect.poll(async () => findSkill(page, skillName, token)).toBeNull(); diff --git a/e2e/specs/mock/unified-upload.spec.ts b/e2e/specs/mock/unified-upload.spec.ts index bb89a05870..fc4ca99df5 100644 --- a/e2e/specs/mock/unified-upload.spec.ts +++ b/e2e/specs/mock/unified-upload.spec.ts @@ -93,7 +93,9 @@ test.describe('unified file upload', () => { expect(((await response.json()) as UploadedFile).llmDeliveryPath).toBe('provider'); // (a) shows as an attachment chip in the composer before sending. - await expect(page.getByRole('button', { name: fileName })).toBeVisible({ timeout: 15000 }); + await expect( + page.getByTestId('composer-tray').getByRole('button', { name: fileName, exact: true }), + ).toBeVisible({ timeout: 15000 }); // (b) reaches the model input: the mock LLM echoes a pass marker only when the // provider file is present in the request content (see e2e/setup/fake-model.js). diff --git a/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts b/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts index 2985c15f91..339244b2c1 100644 --- a/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts @@ -400,12 +400,12 @@ describe('terminal projection boundary', () => { delivered.push(event); }, ); - expect(subscription).not.toBeNull(); - await new Promise((resolve) => setTimeout(resolve, 250)); + // A completed replay may retire its subscription before attachment returns. + await waitFor(() => delivered.length > 0, 'legacy terminal replay'); expect(delivered).toHaveLength(1); expectSafeTerminalEvent(delivered[0]); - subscription!.unsubscribe(); + subscription?.unsubscribe(); } finally { await replica.destroy(); await replicaSubscriber.quit().catch(() => undefined); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 2c9fd2589d..6cf3aba026 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1140,6 +1140,12 @@ const remoteApiAuthSchema = z.object({ const remoteApiSchema = z.object({ auth: remoteApiAuthSchema.optional(), + background: z + .object({ + enabled: z.boolean().default(false), + acknowledgementTimeoutMs: z.number().int().min(100).max(60_000).default(10_000), + }) + .optional(), }); const managementClientBindingSchema = z diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 001156490d..926a56165a 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -39,6 +39,7 @@ export * from './roles'; /* types (exports schemas from `./types` as they contain needed in other defs) */ export * from './types'; export * from './types/agents'; +export * from './types/responseAccounting'; export * from './types/assistants'; export * from './types/content'; export * from './types/tools'; @@ -112,4 +113,6 @@ export * from './types/scheduleConsent'; export * from './approval'; +export * from './types/responseReview'; + export * from './codeSetup'; diff --git a/packages/data-provider/src/types/responseAccounting.ts b/packages/data-provider/src/types/responseAccounting.ts new file mode 100644 index 0000000000..7ed4ed0a40 --- /dev/null +++ b/packages/data-provider/src/types/responseAccounting.ts @@ -0,0 +1,45 @@ +/** An application-billing amount is not a provider invoice or list-price estimate. */ +export type AccountingBasis = 'provider_reported' | 'application_billing'; +export type AccountingModelSource = + | 'provider_reported' + | 'runtime_reported' + | 'configured' + | 'unknown'; + +export interface ResponseAccountingCall { + provider?: string; + model?: string; + modelSource: AccountingModelSource; + usageType: + | 'message' + | 'summarization' + | 'subagent' + | 'sequential' + | 'activity-label' + | 'activity-phase' + | 'reasoning-label'; + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + amount?: string; + basis?: AccountingBasis; + reason: 'priced' | 'pricing_unavailable' | 'usage_unavailable' | 'invalid_usage'; +} + +/** Separate from token usage: partial costs must never appear as a run's total. */ +export interface ResponseAccounting { + status: 'available' | 'unavailable'; + currency: 'USD'; + amount?: string; + basis?: AccountingBasis; + coverage: 'complete' | 'partial' | 'none'; + reason: + | 'priced' + | 'pricing_unavailable' + | 'usage_unavailable' + | 'partial_coverage' + | 'mixed_bases' + | 'conflicting_usage'; + calls: ResponseAccountingCall[]; +} diff --git a/packages/data-provider/src/types/responseReview.ts b/packages/data-provider/src/types/responseReview.ts new file mode 100644 index 0000000000..249e87c079 --- /dev/null +++ b/packages/data-provider/src/types/responseReview.ts @@ -0,0 +1,55 @@ +import { z } from 'zod'; +import type { Agents } from './agents'; + +export const responseReviewDecisionSchema = z + .object({ + actionId: z.string().min(1).max(256), + answer: z.string().optional(), + answers: z.record(z.string(), z.string()).optional(), + decisions: z + .array( + z + .object({ + tool_call_id: z.string(), + decision: z.enum(['approve', 'reject', 'edit', 'respond']), + editedArguments: z.record(z.string(), z.unknown()).optional(), + responseText: z.string().optional(), + reason: z.string().optional(), + scope: z.enum(['once', 'session', 'always']).optional(), + }) + .strict(), + ) + .optional(), + }) + .strict(); + +export type ResponseReviewDecision = z.infer; +export const responseReviewDecisionReceiptSchema = z.object({ + responseId: z.string().min(1), + actionId: z.string().min(1), + intentId: z.string().min(1), + status: z.enum(['queued', 'settled']), +}); +export type ResponseReviewDecisionReceipt = z.infer; +export const responseCancelReceiptSchema = z.object({ + id: z.string().min(1), + status: z.enum(['queued', 'in_progress', 'completed', 'failed', 'incomplete', 'cancelled']), +}); +export type ResponseCancelReceipt = z.infer; + +/** Owner-authenticated browser view; OAuth URLs must never be delivered to a channel. */ +export interface ResponseReviewView { + /** Missing/mismatched execution is not proof that provider work stopped. */ + recovery?: 'execution_state_unavailable'; + responseId: string; + status: 'queued' | 'in_progress' | 'completed' | 'failed' | 'incomplete' | 'cancelled'; + oauth: Array<{ + actionId: string; + toolName: string; + authURL: string; + expiresAt: number; + }>; + pendingAction?: Pick; + pendingDecision?: { intentId: string; decision: ResponseReviewDecision }; + canSubmitDecision?: boolean; +} diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 83b88ff68f..1fec5d0839 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -11,6 +11,22 @@ export * from './crypto'; export * from './schema'; export * from './utils'; export { createModels } from './models'; +export { createResponseStorage } from './methods/response'; +export type { + ResponseOwner, + ResponseStorage, + ResponseCleanupIdentity, + ResponseReservation, + ResponseReservationResult, + ResponseMutationResult, + ResponseReviewCheckpoint, + ResponseResumeIntent, + ResponseResumeReservation, + ResponseResumeResult, + ResponseTerminalReceipt, + StoredResponseRecord, + StoredResponseStatus, +} from './methods/response'; export { createMethods, createSkillDeletionPersistenceAdapter, @@ -103,6 +119,7 @@ export { AUDIT_ACTION_CATEGORY, } from './types/admin'; export { GENESIS_HASH, PLATFORM_CHAIN_KEY } from './schema/auditLog'; + export { default as logger, baseLogFormat } from './config/winston'; export { default as meiliLogger } from './config/meiliLogger'; export { jsonTruncateFormat, redactMessage } from './config/parsers'; diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts index 3f06f1e37a..3063038106 100644 --- a/packages/data-schemas/src/methods/conversation.spec.ts +++ b/packages/data-schemas/src/methods/conversation.spec.ts @@ -20,6 +20,8 @@ import type { import { ConversationMethods, createConversationMethods } from './conversation'; import { tenantStorage, runAsSystem } from '~/config/tenantContext'; import { createChatProjectMethods } from './chatProject'; +import { createResponseStorage } from './response'; +import { waitForPause } from './test-helpers'; import { createModels } from '../models'; jest.mock('~/config/winston', () => ({ @@ -1141,13 +1143,13 @@ describe('Conversation Operations', () => { }, ); + const archive = saveConvo( + { userId: 'user123' }, + { conversationId, isArchived: true }, + { preserveUpdatedAt: true, noUpsert: true }, + ); try { - const archive = saveConvo( - { userId: 'user123' }, - { conversationId, isArchived: true }, - { preserveUpdatedAt: true, noUpsert: true }, - ); - await archiveWriteReached; + await waitForPause(archiveWriteReached, archive); const unarchived = await saveConvo( { userId: 'user123' }, @@ -1166,6 +1168,7 @@ describe('Conversation Operations', () => { } finally { releaseArchiveWrite(); writeSpy.mockRestore(); + await Promise.allSettled([archive]); } }); @@ -3848,6 +3851,150 @@ describe('Conversation Operations', () => { }); describe('deleteConvos', () => { + describe('stored responses on standalone Mongo', () => { + beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope.deleteMany({}); + }); + }); + + async function seedResponse(conversationId: string, tenantId?: string) { + return runAsSystem(() => + mongoose.models.StoredResponse.create({ + userId: 'user123', + tenantId, + conversationId, + responseId: uuidv4(), + requestKey: uuidv4(), + fingerprint: 'request', + agentId: 'agent', + requestJSON: '{"input":"private"}', + responseJSON: '{"output":"private"}', + parentMessageId: 'root', + status: 'completed', + }), + ); + } + + it('rejects background reservation on standalone Mongo before storing private state', async () => { + const result = await createResponseStorage(mongoose).reserve({ + userId: 'user123', + responseId: uuidv4(), + requestKey: uuidv4(), + fingerprint: 'request', + agentId: 'agent', + requestJSON: '{"input":"private"}', + conversationId: uuidv4(), + parentMessageId: 'root', + }); + expect(result).toEqual({ ok: false, code: 'response_unavailable' }); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + expect(await mongoose.models.ResponseScope.countDocuments({})).toBe(0); + }); + + it('erases only the selected tenant and keeps ordinary deletion nontransactional', async () => { + const conversationId = uuidv4(); + await Conversation.create({ + conversationId, + user: 'user123', + tenantId: 'tenant-a', + endpoint: EModelEndpoint.agents, + }); + await seedResponse(conversationId, 'tenant-a'); + await seedResponse(conversationId, 'tenant-b'); + const session = jest.spyOn(mongoose, 'startSession'); + await tenantStorage.run({ tenantId: 'tenant-a' }, () => + deleteConvos('user123', { conversationId }), + ); + expect(session).not.toHaveBeenCalled(); + session.mockRestore(); + await runAsSystem(async () => { + expect( + await mongoose.models.StoredResponse.countDocuments({ tenantId: 'tenant-a' }), + ).toBe(0); + expect( + await mongoose.models.StoredResponse.countDocuments({ tenantId: 'tenant-b' }), + ).toBe(1); + }); + }); + + it('retains payloads when an earlier hook fails but keeps deletion intent closed until retry', async () => { + const conversationId = uuidv4(); + await Conversation.create({ + conversationId, + user: 'user123', + endpoint: EModelEndpoint.agents, + }); + const response = await seedResponse(conversationId); + await expect( + deleteConvos( + 'user123', + { conversationId }, + { + beforeDelete: async () => { + throw new Error('drain failed'); + }, + }, + ), + ).rejects.toThrow('drain failed'); + expect(await Conversation.exists({ conversationId })).not.toBeNull(); + expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(1); + expect( + await createResponseStorage(mongoose).read({ userId: 'user123' }, response.responseId), + ).not.toBeNull(); + await deleteConvos('user123', { conversationId }); + expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0); + }); + + it('recovers private response cleanup after the root and its messages are already gone', async () => { + const conversationId = uuidv4(); + await seedResponse(conversationId); + const result = await deleteConvos('user123', { conversationId }); + expect(result.conversationIds).toContain(conversationId); + expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0); + }); + + it('retries irreversible response erasure when the subsequent parent deletion fails', async () => { + const conversationId = uuidv4(); + await Conversation.create({ + conversationId, + user: 'user123', + endpoint: EModelEndpoint.agents, + }); + await seedResponse(conversationId); + const deletion = jest + .spyOn(Conversation.collection, 'deleteMany') + .mockRejectedValueOnce(new Error('parent delete failed')); + await expect(deleteConvos('user123', { conversationId })).rejects.toThrow( + 'parent delete failed', + ); + deletion.mockRestore(); + expect(await Conversation.exists({ conversationId })).not.toBeNull(); + expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0); + await deleteConvos('user123', { conversationId }); + expect(await Conversation.exists({ conversationId })).toBeNull(); + }); + + it('allows concurrent erasers to finish without reopening admission', async () => { + const conversationId = uuidv4(); + await Conversation.create({ + conversationId, + user: 'user123', + endpoint: EModelEndpoint.agents, + }); + const response = await seedResponse(conversationId); + await Promise.all([ + deleteConvos('user123', { conversationId }, { allowEmpty: true }), + deleteConvos('user123', { conversationId }, { allowEmpty: true }), + ]); + expect(await mongoose.models.StoredResponse.countDocuments({ conversationId })).toBe(0); + expect( + await createResponseStorage(mongoose).read({ userId: 'user123' }, response.responseId), + ).toBeNull(); + }); + }); + it('retires queued-turn work before each conversation deletion wave', async () => { const conversationId = uuidv4(); await Conversation.create({ diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts index 16bbe61175..d24938717d 100644 --- a/packages/data-schemas/src/methods/conversation.ts +++ b/packages/data-schemas/src/methods/conversation.ts @@ -29,6 +29,7 @@ import type { IMessage, ISharedLink, ISubagentThreadReservation, + IUser, } from '~/types'; import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili'; import type { MessageMethods } from './message'; @@ -39,6 +40,13 @@ import { MAX_AGENT_EVENT_ACTOR_SUMMARY_LENGTH, MAX_AGENT_EVENT_ACTOR_TOOL_NAME_LENGTH, } from '~/types/convo'; +import { + closeResponseScopes, + eraseResponseRecords, + createResponseRetention, + discoverResponseOwners, + createResponseConversationReplacement, +} from './responseScope'; import { activeExpirationFilter, buildRetentionVisibilityFilter, @@ -49,6 +57,7 @@ import { updateChatProjectLastConversationForUser, } from './chatProject'; import { createChatExpirationDate, createTempChatExpirationDate } from '~/utils/tempChatRetention'; +import { getTenantId, SYSTEM_TENANT_ID, runAsSystem } from '~/config/tenantContext'; import { isAgentFadingTier, isAgentFadingTierEntries } from '~/utils/fading'; import { isCompactionSemanticIndexProjection } from '~/types/compaction'; import { withoutMeiliIndexing } from '~/models/plugins/mongoMeili'; @@ -668,6 +677,8 @@ export function createConversationMethods( mongoose: typeof import('mongoose'), deps?: ConversationMethodDeps, ): ConversationMethods { + const retainResponses = createResponseRetention(mongoose); + const prepareResponseReplacement = createResponseConversationReplacement(mongoose); let legacyReceiptExpiryCursor: Types.ObjectId | undefined; /** @@ -2336,12 +2347,13 @@ export function createConversationMethods( const Conversation = mongoose.models.Conversation as Model; const stored = await Conversation.findOne({ conversationId, user: userId }) - .select({ _id: 1, isTemporary: 1, expiredAt: 1, tags: 1 }) + .select({ _id: 1, isTemporary: 1, expiredAt: 1, tags: 1, tenantId: 1 }) .lean<{ _id: Types.ObjectId; isTemporary?: boolean; expiredAt?: Date | null; tags?: string[]; + tenantId?: string; } | null>(); if (!stored) { return; @@ -2357,6 +2369,7 @@ export function createConversationMethods( } } + await retainResponses({ userId, tenantId: stored.tenantId }, conversationId, expiredAt); let stampedDeadline = stored.expiredAt != null; if (stored.isTemporary !== true) { /** @@ -2580,6 +2593,37 @@ export function createConversationMethods( update.expiredAt = null; } + const responseDeadline = + update.expiredAt instanceof Date ? update.expiredAt : retentionOnInsert?.expiredAt; + const replacingConversationId = newConversationId && newConversationId !== conversationId; + if (responseDeadline || replacingConversationId) { + const stored = await Conversation.findOne({ user: userId, conversationId }) + .select('tenantId expiredAt') + .lean>(); + const contextTenant = getTenantId(); + const responseOwner = { + userId, + tenantId: + stored?.tenantId ?? (contextTenant === SYSTEM_TENANT_ID ? undefined : contextTenant), + }; + // Conversation IDs are shared foreign keys for messages, approvals, jobs + // and checkpoints. newConversationId assigns an initial ID, never renames + // a persisted conversation independently of those resources. + if ( + replacingConversationId && + (stored || !(await prepareResponseReplacement(responseOwner, conversationId))) + ) { + return { message: 'Conversation ID cannot be replaced after persistence.' }; + } + const deadline = responseDeadline ?? stored?.expiredAt; + if (deadline) + await retainResponses( + responseOwner, + newConversationId || conversationId, + stored?.expiredAt && stored.expiredAt < deadline ? stored.expiredAt : deadline, + ); + } + const createdAtOnInsert = metadata?.createdAtOnInsert instanceof Date && !Number.isNaN(metadata.createdAtOnInsert.getTime()) @@ -2652,6 +2696,9 @@ export function createConversationMethods( const baseFilter = { conversationId, user: userId, + // Initial ID assignment must never match a source inserted after the + // absence check. Upsert may create the new ID, but cannot rename that row. + ...(replacingConversationId ? { _id: { $exists: false } } : {}), ...(metadata?.titleSource === 'generated' && { titleSetByUser: { $ne: true }, title: { $in: [null, '', 'New Chat'] }, @@ -2773,6 +2820,11 @@ export function createConversationMethods( interfaceConfig, conversation.isTemporary === true, ); + await retainResponses( + { userId, tenantId: conversation.tenantId }, + conversationId, + deadline, + ); const result = await Conversation.updateOne( { _id: conversation._id, @@ -4046,12 +4098,40 @@ export function createConversationMethods( .select('conversationId tenantId chatProjectId tags') .lean(); const recoveryConversationIds: string[] = []; + const recoveryResponseScopes: DeletionConversation[] = []; + const responseGroups = (wave: DeletionConversation[]) => { + const groups = new Map(); + for (const conversation of wave) { + const tenantId = conversation.tenantId || undefined; + const ids = groups.get(tenantId) ?? []; + ids.push(conversation.conversationId); + groups.set(tenantId, ids); + } + return groups; + }; + const closeResponses = async (wave: DeletionConversation[]) => { + for (const [tenantId, ids] of responseGroups(wave)) + await closeResponseScopes(mongoose, { userId: user, tenantId }, ids); + }; + const eraseResponses = async (wave: DeletionConversation[]) => { + for (const [tenantId, ids] of responseGroups(wave)) + await eraseResponseRecords(mongoose, { userId: user, tenantId }, ids); + }; if (!conversations.length && typeof filter.conversationId === 'string') { + // Authorized delayed reservations retain the globally unique user's actual tenant. + // Fence it before probing legacy payloads, including when no response exists yet. + const owner = isValidObjectIdString(user) + ? await (mongoose.models.User as Model).findById(user).select('tenantId').lean() + : null; + if (owner) + await closeResponses([ + { conversationId: filter.conversationId, tenantId: owner.tenantId }, + ]); /** A prior attempt may have deleted the root before a descendant read failed. * Resume from immutable root lineage and retain the root id for message, * checkpoint, and tool cleanup. The message probe distinguishes that partial * commit from a conversation id that never existed. */ - const [descendants, rootMessages] = await Promise.all([ + const [descendants, rootMessages, responseOwners] = await Promise.all([ retryCascadeOperation(() => Conversation.find({ user, @@ -4061,8 +4141,20 @@ export function createConversationMethods( .lean(), ), getMessages({ user, conversationId: filter.conversationId }, '_id', { limit: 1 }), + discoverResponseOwners(mongoose, user, filter.conversationId), ]); - if (descendants.length === 0 && rootMessages.length === 0 && options?.allowEmpty !== true) { + for (const responseOwner of responseOwners) { + recoveryResponseScopes.push({ + conversationId: filter.conversationId, + tenantId: responseOwner.tenantId, + }); + } + if ( + descendants.length === 0 && + rootMessages.length === 0 && + recoveryResponseScopes.length === 0 && + options?.allowEmpty !== true + ) { throw new Error('Conversation not found or already deleted.'); } conversations = descendants; @@ -4132,6 +4224,7 @@ export function createConversationMethods( break; } const waveIds = wave.map((conversation) => conversation.conversationId); + await closeResponses(wave); await deps?.deleteAgentQueuedTurns?.( user, wave.map((conversation) => ({ @@ -4147,6 +4240,8 @@ export function createConversationMethods( conversationId: { $in: waveIds }, }), ); + // Like subagent content, response erasure is an irreversible part of deletion intent. + await eraseResponses(wave); const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } }); if (result.deletedCount > 0) { /** Result erasure is irreversible. Keep receipts intact when a @@ -4181,6 +4276,12 @@ export function createConversationMethods( ]; if (recoveryConversationIds.length > 0) { + // Each command retains an explicit owner/partition predicate. Request tenant + // middleware must not replace the partitions discovered by recovery. + await runAsSystem(async () => { + await closeResponses(recoveryResponseScopes); + await eraseResponses(recoveryResponseScopes); + }); await deps?.deleteAgentQueuedTurns?.( user, recoveryConversationIds.map((conversationId) => ({ diff --git a/packages/data-schemas/src/methods/import.spec.ts b/packages/data-schemas/src/methods/import.spec.ts index b83f25bbb8..ef994f49b9 100644 --- a/packages/data-schemas/src/methods/import.spec.ts +++ b/packages/data-schemas/src/methods/import.spec.ts @@ -69,6 +69,20 @@ describe('conversation import cleanup methods', () => { text: record.conversationId, })), ); + await mongoose.models.StoredResponse.insertMany( + records.map((record, index) => ({ + userId: record.user, + tenantId: record.tenantId, + conversationId: record.conversationId, + responseId: `response-${index}`, + requestKey: `request-${index}`, + fingerprint: 'request', + agentId: 'agent', + requestJSON: '{"input":"private"}', + parentMessageId: 'root', + status: 'queued', + })), + ); await mongoose.models.ChatProject.updateOne( { _id: project._id }, { @@ -99,6 +113,9 @@ describe('conversation import cleanup methods', () => { }); expect(remaining.conversations).toHaveLength(3); expect(remaining.messages).toHaveLength(3); + expect(await runAsSystem(async () => mongoose.models.StoredResponse.countDocuments({}))).toBe( + 3, + ); expect( remaining.conversations.some( (record) => @@ -158,10 +175,17 @@ describe('conversation import cleanup methods', () => { const deleteMessages = jest.fn().mockResolvedValue(undefined); const findProjects = jest.fn().mockResolvedValue([]); const deleteConversations = jest.fn().mockResolvedValue(undefined); + const responseScopes = { bulkWrite: jest.fn().mockResolvedValue(undefined) }; const cleanupMongoose = { + model: jest.fn().mockReturnValue(responseScopes), models: { Message: { deleteMany: deleteMessages }, Conversation: { distinct: findProjects, deleteMany: deleteConversations }, + ResponseScope: responseScopes, + StoredResponse: { + updateMany: jest.fn().mockResolvedValue(undefined), + deleteMany: jest.fn().mockResolvedValue(undefined), + }, }, } as unknown as typeof mongoose; const methods = createConversationImportMethods(cleanupMongoose); diff --git a/packages/data-schemas/src/methods/import.ts b/packages/data-schemas/src/methods/import.ts index 068704200b..4fd5b46b24 100644 --- a/packages/data-schemas/src/methods/import.ts +++ b/packages/data-schemas/src/methods/import.ts @@ -1,5 +1,6 @@ import type { FilterQuery, Model } from 'mongoose'; import type { IConversation, IMessage } from '~/types'; +import { closeResponseScopes, eraseResponseRecords } from './responseScope'; import { runAsSystem, tenantStorage } from '~/config/tenantContext'; import { refreshChatProjectStatsForUser } from './chatProject'; @@ -77,6 +78,9 @@ export function createConversationImportMethods( affectedProjectIds.add(String(projectId)); } } + const owner = { userId: scope.user, tenantId: scope.tenantId }; + await closeResponseScopes(mongoose, owner, conversationIds); + await eraseResponseRecords(mongoose, owner, conversationIds); await Conversation.deleteMany(filter); await mongoose.models.SubagentContent?.deleteMany(filter); } diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index 0e17067550..f09c8dda87 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -67,6 +67,7 @@ import { import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCategory'; /* Agent API Keys */ import { createAgentApiKeyMethods, type AgentApiKeyMethods } from './agentApiKey'; + /* MCP Servers */ import { createMCPServerMethods, type MCPServerMethods } from './mcpServer'; import { createCodeEnvironmentMethods, type CodeEnvironmentMethods } from './codeEnvironment'; diff --git a/packages/data-schemas/src/methods/response.spec.ts b/packages/data-schemas/src/methods/response.spec.ts new file mode 100644 index 0000000000..0c62d09fb2 --- /dev/null +++ b/packages/data-schemas/src/methods/response.spec.ts @@ -0,0 +1,1519 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import type { + ResponseReservation, + ResponseStorage, + ResponseResumeReservation, + ResponseTerminalReceipt, +} from './response'; +import { closeResponseScopes, createResponseRetention } from './responseScope'; +import { tenantStorage, runAsSystem } from '~/config/tenantContext'; +import { createConversationMethods } from './conversation'; +import { createConversationImportMethods } from './import'; +import { createResponseStorage } from './response'; +import { waitForPause } from './test-helpers'; +import { createModels } from '~/models'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +const owner = { userId: 'owner', tenantId: 'tenant-a' }; +const scoped = (work: () => Promise) => tenantStorage.run({ tenantId: owner.tenantId }, work); +const reservation = (overrides: Partial = {}): ResponseReservation => ({ + ...owner, + responseId: 'resp_one', + requestKey: 'request-one', + fingerprint: 'same-request', + agentId: 'agent-one', + requestJSON: JSON.stringify({ + model: 'agent-one', + input: 'Hello', + background: true, + store: true, + }), + conversationId: 'conversation-one', + parentMessageId: 'root', + ...overrides, +}); + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + storage = createResponseStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); +}); + +it('creates correctness indexes with autoIndex disabled and deduplicates concurrent requests', () => + scoped(async () => { + const results = await Promise.all( + Array.from({ length: 12 }, (_, index) => + storage.reserve(reservation({ responseId: `resp_${index}` })), + ), + ); + expect(results.every((result) => result.ok)).toBe(true); + const records = results.flatMap((result) => (result.ok ? [result.record.responseId] : [])); + expect(new Set(records).size).toBe(1); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + expect(results.filter((result) => result.ok && !result.replayed)).toHaveLength(1); + })); + +it('expires private reads and mutations immediately while retaining only cleanup identity', () => + scoped(async () => { + const expiredAt = new Date(Date.now() + 60_000); + await storage.reserve(reservation({ expiredAt })); + expect(await mongoose.models.StoredResponse.collection.indexExists('expiredAt_1')).toBe(true); + await createResponseRetention(mongoose)(owner, 'conversation-one', new Date(Date.now() - 1)); + expect(await storage.read(owner, 'resp_one')).toBeNull(); + expect(await storage.requestCancellation(owner, 'resp_one', Date.now())).toBeNull(); + expect(await storage.attachDelivery(owner, 'resp_one', 'late')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect( + await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + responseJSON: '{"late":"private"}', + generationCreatedAt: 1, + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + const identity = await storage.readCleanupIdentity(owner, 'resp_one'); + expect(identity).toMatchObject({ responseId: 'resp_one', userId: owner.userId }); + expect(JSON.stringify(identity)).not.toContain('Hello'); + expect(identity).not.toHaveProperty('requestJSON'); + expect(identity).not.toHaveProperty('responseJSON'); + await mongoose.models.StoredResponse.deleteMany({}); + expect(await storage.reserve(reservation())).toEqual({ + ok: false, + code: 'response_unavailable', + }); + })); + +it('propagates conversation retention before parent persistence and never lengthens response expiry', () => + scoped(async () => { + createModels(mongoose); + await storage.reserve(reservation()); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + const deadline = new Date(Date.now() + 60_000); + await methods.saveConvo( + { userId: owner.userId, expiredAt: deadline }, + { + conversationId: 'conversation-one', + endpoint: 'agents', + }, + ); + expect((await storage.read(owner, 'resp_one'))?.expiredAt).toEqual(deadline); + await methods.saveConvo( + { userId: owner.userId, expiredAt: new Date(Date.now() + 120_000) }, + { + conversationId: 'conversation-one', + endpoint: 'agents', + }, + ); + expect((await storage.read(owner, 'resp_one'))?.expiredAt).toEqual(deadline); + await createResponseRetention(mongoose)(owner, 'conversation-one', new Date(Date.now() - 1)); + expect(await storage.reserve(reservation({ requestKey: 'late' }))).toEqual({ + ok: false, + code: 'response_unavailable', + }); + })); + +it('applies newly resolved expiry to replay and sibling payloads in the reservation transaction', () => + scoped(async () => { + await storage.reserve(reservation()); + await storage.reserve(reservation({ responseId: 'resp_sibling', requestKey: 'sibling' })); + const expiredAt = new Date(Date.now() + 60_000); + const replay = await storage.reserve(reservation({ expiredAt })); + expect(replay).toMatchObject({ ok: true, replayed: true, record: { expiredAt } }); + expect((await storage.read(owner, 'resp_sibling'))?.expiredAt).toEqual(expiredAt); + expect(await storage.reserve(reservation({ expiredAt: new Date(Date.now() - 1) }))).toEqual({ + ok: false, + code: 'response_unavailable', + }); + expect(await storage.read(owner, 'resp_one')).toBeNull(); + expect(await storage.read(owner, 'resp_sibling')).toBeNull(); + })); + +it('rejects changed input under the same idempotency key without replacing the original', () => + scoped(async () => { + await storage.reserve(reservation()); + expect( + await storage.reserve(reservation({ fingerprint: 'changed', requestJSON: 'changed' })), + ).toEqual({ ok: false, code: 'idempotency_conflict' }); + expect((await storage.read(owner, 'resp_one'))?.fingerprint).toBe('same-request'); + })); + +it('replays concurrent retries against the winning conversation rather than their fresh ids', () => + scoped(async () => { + const results = await Promise.all( + Array.from({ length: 8 }, (_, index) => + storage.reserve( + reservation({ + responseId: `resp_${index}`, + conversationId: `conversation-${index}`, + }), + ), + ), + ); + expect(results.every((result) => result.ok)).toBe(true); + expect( + new Set(results.flatMap((result) => (result.ok ? [result.record.conversationId] : []))).size, + ).toBe(1); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + })); + +it('rolls back admission writes when insertion fails before commit', () => + scoped(async () => { + jest + .spyOn(mongoose.models.StoredResponse.collection, 'insertOne') + .mockRejectedValueOnce(new Error('interrupted insertion')); + await expect(storage.reserve(reservation())).rejects.toThrow('interrupted insertion'); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + const anchors = await mongoose.models.ResponseScope.find({}).lean(); + expect(anchors.every((anchor) => anchor.revision === 0)).toBe(true); + await closeResponseScopes(mongoose, owner); + expect(await storage.reserve(reservation())).toEqual({ + ok: false, + code: 'response_unavailable', + }); + })); + +it('fences an authenticated delayed reservation before missing-root recovery probes', async () => { + createModels(mongoose); + const user = await runAsSystem(() => + mongoose.models.User.create({ + email: 'response-recovery@example.com', + provider: 'local', + tenantId: owner.tenantId, + }), + ); + const authenticated = { userId: String(user._id), tenantId: user.tenantId }; + const conversationId = 'already-deleted-root'; + let delayedResult; + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => { + // This request passed authority before recovery, then reached reserve after its probes began. + delayedResult = await storage.reserve(reservation({ ...authenticated, conversationId })); + return []; + }), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + await runAsSystem(() => + methods.deleteConvos(authenticated.userId, { conversationId }, { allowEmpty: true }), + ); + expect(delayedResult).toEqual({ ok: false, code: 'response_unavailable' }); + expect(await storage.read(authenticated, 'resp_one')).toBeNull(); + await runAsSystem(() => mongoose.models.User.deleteOne({ _id: user._id })); +}); + +it.each([undefined, ''])( + 'shares tenantless admission and reads for tenantId=%p', + async (tenantId) => { + const tenantless = { userId: 'tenantless', tenantId }; + await tenantStorage.run({ tenantId: '' }, async () => { + const result = await storage.reserve(reservation(tenantless)); + expect(result.ok).toBe(true); + if (result.ok) expect(result.record.tenantId).toBe(tenantId); + expect(await storage.read({ userId: tenantless.userId }, 'resp_one')).not.toBeNull(); + expect( + await storage.reserve( + reservation({ ...tenantless, tenantId: '', responseId: 'resp_retry' }), + ), + ).toMatchObject({ ok: true, replayed: true }); + await mongoose.models.StoredResponse.collection.updateOne( + { responseId: 'resp_one' }, + { $set: { tenantId: null } }, + ); + expect(await storage.read({ ...tenantless, tenantId: '' }, 'resp_one')).not.toBeNull(); + await closeResponseScopes(mongoose, { ...tenantless, tenantId: '' }); + expect( + await storage.reserve( + reservation({ userId: tenantless.userId, tenantId: undefined, requestKey: 'later' }), + ), + ).toEqual({ + ok: false, + code: 'response_unavailable', + }); + }); + }, +); + +it('fences an in-flight response before import rollback deletes its conversation', () => + scoped(async () => { + createModels(mongoose); + await mongoose.models.Conversation.create({ + user: owner.userId, + conversationId: 'conversation-one', + endpoint: 'agents', + }); + let entered!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const collection = mongoose.models.StoredResponse.collection; + const insert = collection.insertOne.bind(collection); + const paused = jest.spyOn(collection, 'insertOne').mockImplementationOnce(async (...args) => { + entered(); + await gate; + return insert(...args); + }); + const pending = storage.reserve(reservation()); + await waitForPause(started, pending); + const rollback = createConversationImportMethods(mongoose).deleteImportedConversations({ + user: owner.userId, + tenantId: owner.tenantId, + conversationIds: ['conversation-one'], + }); + const drained = Promise.allSettled([pending, rollback]); + try { + await new Promise((resolve) => setTimeout(resolve, 50)); + expect( + await mongoose.models.Conversation.exists({ conversationId: 'conversation-one' }), + ).not.toBeNull(); + } finally { + release(); + paused.mockRestore(); + await drained; + } + expect((await pending).ok).toBe(true); + await rollback; + expect(await storage.read(owner, 'resp_one')).toBeNull(); + expect( + await mongoose.models.Conversation.exists({ conversationId: 'conversation-one' }), + ).toBeNull(); + expect(await storage.reserve(reservation())).toEqual({ + ok: false, + code: 'response_unavailable', + }); + })); + +it('does not reveal or mutate a response belonging to another user or tenant', () => + scoped(async () => { + await storage.reserve(reservation()); + for (const stranger of [ + { ...owner, userId: 'stranger' }, + { ...owner, tenantId: 'tenant-b' }, + ]) { + await tenantStorage.run({ tenantId: stranger.tenantId }, async () => { + expect(await storage.read(stranger, 'resp_one')).toBeNull(); + expect(await storage.attachDelivery(stranger, 'resp_one', 'delivery')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect( + await storage.publish(stranger, 'resp_one', 0, { + status: 'completed', + responseJSON: '{}', + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect(await storage.requestCancellation(stranger, 'resp_one', 100)).toBeNull(); + }); + } + expect((await storage.read(owner, 'resp_one'))?.revision).toBe(0); + })); + +it('rejects an explicit owner tenant that disagrees with authenticated async context', () => + scoped(async () => { + await expect(storage.reserve(reservation({ tenantId: 'tenant-b' }))).rejects.toThrow( + 'Response tenant does not match authenticated scope', + ); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + })); + +it('allows the same request key independently for another authenticated owner', () => + scoped(async () => { + expect((await storage.reserve(reservation())).ok).toBe(true); + expect( + (await storage.reserve(reservation({ userId: 'other', responseId: 'resp_other' }))).ok, + ).toBe(true); + })); + +it('attaches only one immutable queue delivery identity', () => + scoped(async () => { + await storage.reserve(reservation()); + expect((await storage.attachDelivery(owner, 'resp_one', 'delivery')).ok).toBe(true); + expect((await storage.attachDelivery(owner, 'resp_one', 'delivery')).ok).toBe(true); + expect(await storage.attachDelivery(owner, 'resp_one', 'different')).toEqual({ + ok: false, + code: 'response_conflict', + }); + })); + +it('CAS-publishes one terminal result and never allows a stale observer to clobber it', () => + scoped(async () => { + await storage.reserve(reservation()); + const results = await Promise.all( + ['completed', 'cancelled'].map((status) => + storage.publish(owner, 'resp_one', 0, { + status: status as 'completed' | 'cancelled', + responseJSON: status, + }), + ), + ); + expect(results.filter((result) => result.ok)).toHaveLength(1); + const record = (await storage.read(owner, 'resp_one'))!; + expect( + await storage.publish(owner, 'resp_one', record.revision, { + status: 'in_progress', + responseJSON: 'stale', + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect((await storage.read(owner, 'resp_one'))?.responseJSON).toBe(record.responseJSON); + })); + +it('rejects a replacement generation and backwards queued transitions', () => + scoped(async () => { + await storage.reserve(reservation()); + expect( + ( + await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + responseJSON: '{}', + generationCreatedAt: 10, + }) + ).ok, + ).toBe(true); + expect( + await storage.publish(owner, 'resp_one', 1, { + status: 'completed', + responseJSON: '{}', + generationCreatedAt: 20, + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect( + await storage.publish(owner, 'resp_one', 1, { status: 'queued', responseJSON: '{}' }), + ).toEqual({ ok: false, code: 'response_conflict' }); + })); + +it('persists cancellation intent once without claiming the provider has stopped', () => + scoped(async () => { + await storage.reserve(reservation()); + await storage.requestCancellation(owner, 'resp_one', 100); + const record = await storage.requestCancellation(owner, 'resp_one', 200); + expect(record).toMatchObject({ status: 'queued', cancellationRequestedAt: 100, revision: 0 }); + })); + +it('rejects queued execution after cancellation without relying on a revision change', () => + scoped(async () => { + await storage.reserve(reservation()); + await storage.requestCancellation(owner, 'resp_one', 100); + expect( + await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: '{}', + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + })); + +it('requires a usable initial generation and permits subsequent progress without repeating it', () => + scoped(async () => { + await storage.reserve(reservation()); + expect( + await storage.publish(owner, 'resp_one', 0, { status: 'in_progress', responseJSON: '{}' }), + ).toEqual({ ok: false, code: 'response_conflict' }); + for (const generationCreatedAt of [0, -1, NaN, Infinity, 1.5]) + await expect( + storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt, + responseJSON: '{}', + }), + ).rejects.toThrow('Invalid response generation identity'); + expect( + ( + await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: '{}', + }) + ).ok, + ).toBe(true); + await storage.requestCancellation(owner, 'resp_one', 100); + expect( + ( + await storage.publish(owner, 'resp_one', 1, { + status: 'in_progress', + responseJSON: '{"progress":true}', + }) + ).ok, + ).toBe(true); + })); + +it('does not add cancellation intent to an already completed response', () => + scoped(async () => { + await storage.reserve(reservation()); + await storage.publish(owner, 'resp_one', 0, { status: 'completed', responseJSON: '{}' }); + expect( + (await storage.requestCancellation(owner, 'resp_one', 100))?.cancellationRequestedAt, + ).toBeUndefined(); + })); + +const review = { + actionId: 'action-one', + providerExecutionId: 'provider-one', + expiresAt: 1000, + partialResponseJSON: '{"private":"checkpoint seed"}', +}; +const resume = (overrides: Partial = {}): ResponseResumeReservation => ({ + generationCreatedAt: 10, + intentId: 'intent-one', + actionId: review.actionId, + previousProviderExecutionId: review.providerExecutionId, + providerExecutionId: 'provider-two', + fingerprint: 'decision-one', + resolutionJSON: '{"approved":true}', + ...overrides, +}); + +async function pause() { + await storage.reserve(reservation()); + const started = await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + responseJSON: '{}', + generationCreatedAt: 10, + }); + expect(started.ok).toBe(true); + const paused = await storage.publishReview( + owner, + 'resp_one', + 1, + 10, + review, + '{"status":"requires_action"}', + ); + expect(paused.ok).toBe(true); +} + +it('retains a valid private visible-token seed and rejects fractional, negative or overflowing counters', () => + scoped(async () => { + await pause(); + const result = await storage.publishReview( + owner, + 'resp_one', + 2, + 10, + { ...review, visibleOutputTokens: 13 }, + '{"status":"requires_action"}', + ); + expect(result).toMatchObject({ ok: true, record: { review: { visibleOutputTokens: 13 } } }); + if (!result.ok) throw new Error('Review publication failed'); + expect(result.record.responseJSON).not.toContain('visibleOutputTokens'); + for (const visibleOutputTokens of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) + await expect( + storage.publishReview( + owner, + 'resp_one', + result.record.revision, + 10, + { ...review, visibleOutputTokens }, + '{}', + ), + ).rejects.toThrow(); + expect((await storage.read(owner, 'resp_one'))?.review?.visibleOutputTokens).toBe(13); + })); + +it('persists one immutable resume decision under concurrent equivalent retries', () => + scoped(async () => { + await pause(); + const results = await Promise.all( + Array.from({ length: 12 }, (_, index) => + storage.reserveResume( + owner, + 'resp_one', + resume({ + intentId: `intent-${index}`, + providerExecutionId: `next-${index}`, + }), + 100, + ), + ), + ); + expect(results.every((result) => result.ok)).toBe(true); + expect(results.filter((result) => result.ok && !result.replayed)).toHaveLength(1); + const intents = results.flatMap((result) => + result.ok ? [result.record.resume?.intentId] : [], + ); + expect(new Set(intents).size).toBe(1); + expect( + await storage.reserveResume( + owner, + 'resp_one', + resume({ resolutionJSON: '{"approved":false}' }), + 100, + ), + ).toEqual({ ok: false, code: 'resume_conflict' }); + const stored = (await storage.read(owner, 'resp_one'))!; + expect(stored.requestJSON).toBe(reservation().requestJSON); + expect(stored.review).toEqual(review); + expect(stored.revision).toBe(3); + })); + +it('retains the original wire decision and does not conflate different scopes with the same SDK resolution', () => + scoped(async () => { + await pause(); + expect( + await storage.reserveResume( + owner, + 'resp_one', + resume({ decisionJSON: '{"scope":"once"}' }), + 500, + ), + ).toMatchObject({ ok: true, record: { resume: { decisionJSON: '{"scope":"once"}' } } }); + expect( + await storage.reserveResume( + owner, + 'resp_one', + resume({ decisionJSON: '{"scope":"session"}' }), + 500, + ), + ).toMatchObject({ ok: false, code: 'resume_conflict' }); + expect((await storage.read(owner, 'resp_one'))!.resume?.decisionJSON).toBe('{"scope":"once"}'); + })); + +it('rejects expired, wrong-action, wrong-generation and wrong-provider decisions', () => + scoped(async () => { + await pause(); + await expect( + storage.reserveResume( + owner, + 'resp_one', + resume({ + providerExecutionId: review.providerExecutionId, + }), + 100, + ), + ).rejects.toThrow('Resume requires a new provider execution identity'); + for (const overrides of [ + { actionId: 'other' }, + { generationCreatedAt: 11 }, + { previousProviderExecutionId: 'other' }, + ]) { + expect(await storage.reserveResume(owner, 'resp_one', resume(overrides), 100)).toEqual({ + ok: false, + code: 'resume_conflict', + }); + } + expect(await storage.reserveResume(owner, 'resp_one', resume(), review.expiresAt)).toEqual({ + ok: false, + code: 'resume_conflict', + }); + expect((await storage.read(owner, 'resp_one'))?.resume).toBeUndefined(); + })); + +it('keeps review and resume mutations owner- and tenant-scoped', () => + scoped(async () => { + await pause(); + for (const stranger of [ + { ...owner, userId: 'other' }, + { ...owner, tenantId: 'other' }, + ]) { + await tenantStorage.run({ tenantId: stranger.tenantId }, async () => { + expect(await storage.publishReview(stranger, 'resp_one', 2, 10, review, '{}')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect(await storage.reserveResume(stranger, 'resp_one', resume(), 100)).toEqual({ + ok: false, + code: 'resume_conflict', + }); + expect( + await storage.admitResume( + stranger, + 'resp_one', + 'intent-one', + 10, + 'provider-two', + '{}', + 100, + ), + ).toEqual({ ok: false, code: 'response_conflict' }); + }); + } + expect((await storage.read(owner, 'resp_one'))?.revision).toBe(2); + })); + +it('fences late pause callbacks and allows only the admitted next segment to pause again', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + expect(await storage.publishReview(owner, 'resp_one', 3, 10, review, '{}')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect( + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'wrong-provider', '{}', 101), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect( + (await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101)) + .ok, + ).toBe(true); + const admitted = (await storage.read(owner, 'resp_one'))!; + expect(admitted).toMatchObject({ + status: 'in_progress', + revision: 4, + resume: { state: 'admitted', admittedAt: 101 }, + }); + expect( + ( + await storage.admitResume( + owner, + 'resp_one', + 'intent-one', + 10, + 'provider-two', + 'ignored', + 102, + ) + ).ok, + ).toBe(true); + expect(await storage.publishReview(owner, 'resp_one', 4, 10, review, '{}')).toEqual({ + ok: false, + code: 'response_conflict', + }); + const next = { ...review, actionId: 'action-two', providerExecutionId: 'provider-two' }; + expect((await storage.publishReview(owner, 'resp_one', 4, 10, next, '{}')).ok).toBe(true); + expect(await storage.read(owner, 'resp_one')).toMatchObject({ + status: 'requires_action', + review: next, + revision: 5, + }); + expect((await storage.read(owner, 'resp_one'))?.resume).toBeUndefined(); + expect(await storage.reserveResume(owner, 'resp_one', resume(), 103)).toEqual({ + ok: false, + code: 'resume_conflict', + }); + })); + +it('cancellation prevents a reserved decision from admitting execution', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + await storage.requestCancellation(owner, 'resp_one', 101); + expect( + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 102), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect(await storage.publishReview(owner, 'resp_one', 3, 10, review, '{}')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect(await storage.reserveResume(owner, 'resp_one', resume(), 103)).toMatchObject({ + ok: true, + replayed: true, + record: { status: 'requires_action', cancellationRequestedAt: 101 }, + }); + })); + +it('does not admit or repause terminal responses but retains the identical decision receipt', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + expect( + (await storage.publish(owner, 'resp_one', 3, { status: 'cancelled', responseJSON: '{}' })).ok, + ).toBe(true); + expect( + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 102), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect(await storage.publishReview(owner, 'resp_one', 4, 10, review, '{}')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect(await storage.reserveResume(owner, 'resp_one', resume(), 103)).toMatchObject({ + ok: true, + replayed: true, + record: { status: 'cancelled', revision: 4 }, + }); + })); + +const outcome: ResponseTerminalReceipt = { + generationCreatedAt: 10, + providerExecutionId: 'provider-one', + ownerId: 'worker-one', + status: 'completed', + responseJSON: '{"final":"private until cleanup"}', + checkpointNamespace: 'owned-checkpoint', +}; + +it('stages an initial terminal receipt without requiring a review or resume', () => + scoped(async () => { + await storage.reserve(reservation()); + await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + responseJSON: '{}', + generationCreatedAt: 10, + }); + expect((await storage.stageTerminal(owner, 'resp_one', 1, outcome)).ok).toBe(true); + expect((await storage.finishTerminal(owner, 'resp_one', 2, outcome)).ok).toBe(true); + })); + +it('stages the admitted provider terminal receipt without requiring another review', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101); + const current = { ...outcome, providerExecutionId: 'provider-two' }; + expect((await storage.stageTerminal(owner, 'resp_one', 4, current)).ok).toBe(true); + expect((await storage.finishTerminal(owner, 'resp_one', 5, current)).ok).toBe(true); + })); + +it('keeps a paused response behind resume admission even with its latest revision', () => + scoped(async () => { + await pause(); + expect( + await storage.publish(owner, 'resp_one', 2, { + status: 'in_progress', + responseJSON: '{}', + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect(await storage.read(owner, 'resp_one')).toMatchObject({ + status: 'requires_action', + revision: 2, + review, + }); + expect((await storage.reserveResume(owner, 'resp_one', resume(), 100)).ok).toBe(true); + expect( + (await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101)) + .ok, + ).toBe(true); + })); + +it('rejects terminal receipts from an earlier segment after resume admission and a later pause', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101); + expect(await storage.stageTerminal(owner, 'resp_one', 4, outcome)).toEqual({ + ok: false, + code: 'response_conflict', + }); + const next = { ...review, actionId: 'action-two', providerExecutionId: 'provider-two' }; + await storage.publishReview(owner, 'resp_one', 4, 10, next, '{}'); + expect(await storage.stageTerminal(owner, 'resp_one', 5, outcome)).toEqual({ + ok: false, + code: 'response_conflict', + }); + const current = { ...outcome, providerExecutionId: 'provider-two' }; + expect((await storage.stageTerminal(owner, 'resp_one', 5, current)).ok).toBe(true); + expect((await storage.finishTerminal(owner, 'resp_one', 6, current)).ok).toBe(true); + })); + +it('durably stages one immutable terminal receipt while keeping its output private', () => + scoped(async () => { + await pause(); + const results = await Promise.all( + Array.from({ length: 12 }, () => storage.stageTerminal(owner, 'resp_one', 2, outcome)), + ); + expect(results.every((result) => result.ok)).toBe(true); + const record = (await storage.read(owner, 'resp_one'))!; + expect(record).toMatchObject({ + status: 'requires_action', + revision: 3, + pendingTerminal: outcome, + }); + expect(record.responseJSON).not.toContain('private until cleanup'); + expect( + await storage.stageTerminal(owner, 'resp_one', 3, { ...outcome, status: 'failed' }), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect( + await storage.stageTerminal(owner, 'resp_one', 3, { ...outcome, ownerId: 'other-worker' }), + ).toEqual({ ok: false, code: 'response_conflict' }); + })); + +it('prevents Stop, review and resume writes from overwriting a staged terminal outcome', () => + scoped(async () => { + await pause(); + await storage.stageTerminal(owner, 'resp_one', 2, outcome); + expect( + await storage.publish(owner, 'resp_one', 3, { + status: 'cancelled', + responseJSON: 'overwrite', + }), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect(await storage.publishReview(owner, 'resp_one', 3, 10, review, '{}')).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect(await storage.reserveResume(owner, 'resp_one', resume(), 100)).toEqual({ + ok: false, + code: 'resume_conflict', + }); + expect( + (await storage.requestCancellation(owner, 'resp_one', 100))?.cancellationRequestedAt, + ).toBeUndefined(); + })); + +it('publishes only the exact saved receipt and removes it atomically with terminal publication', () => + scoped(async () => { + await pause(); + await storage.stageTerminal(owner, 'resp_one', 2, outcome); + for (const changed of [ + { ownerId: 'other' }, + { checkpointNamespace: 'other' }, + { checkpointConfigJSON: 'other' }, + { generationCreatedAt: 11 }, + { responseJSON: 'other' }, + { providerExecutionId: 'other' }, + ]) { + expect( + await storage.finishTerminal(owner, 'resp_one', 3, { ...outcome, ...changed }), + ).toEqual({ ok: false, code: 'response_conflict' }); + } + expect((await storage.finishTerminal(owner, 'resp_one', 3, outcome)).ok).toBe(true); + const record = (await storage.read(owner, 'resp_one'))!; + expect(record).toMatchObject({ + status: 'completed', + responseJSON: outcome.responseJSON, + revision: 4, + }); + expect(record.pendingTerminal).toBeUndefined(); + expect(await storage.stageTerminal(owner, 'resp_one', 4, outcome)).toEqual({ + ok: false, + code: 'response_conflict', + }); + })); + +it('does not stage or publish another user or tenant terminal result', () => + scoped(async () => { + await pause(); + await storage.stageTerminal(owner, 'resp_one', 2, outcome); + for (const stranger of [ + { ...owner, userId: 'other' }, + { ...owner, tenantId: 'other' }, + ]) { + await tenantStorage.run({ tenantId: stranger.tenantId }, async () => { + expect(await storage.stageTerminal(stranger, 'resp_one', 3, outcome)).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect(await storage.finishTerminal(stranger, 'resp_one', 3, outcome)).toEqual({ + ok: false, + code: 'response_conflict', + }); + }); + } + expect((await storage.read(owner, 'resp_one'))?.revision).toBe(3); + })); + +it('never initializes a closed owner or conversation scope back to open', () => + scoped(async () => { + await closeResponseScopes(mongoose, owner); + expect(await storage.reserve(reservation())).toEqual({ + ok: false, + code: 'response_unavailable', + }); + await mongoose.models.ResponseScope.deleteMany({}); + await closeResponseScopes(mongoose, owner, ['conversation-one']); + expect(await storage.reserve(reservation())).toEqual({ + ok: false, + code: 'response_unavailable', + }); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + })); + +it('retains the cleanup journal but cannot replay a closed conversation through a fresh continuation', () => + scoped(async () => { + await storage.reserve(reservation()); + await Promise.all([ + closeResponseScopes(mongoose, owner, ['conversation-one']), + closeResponseScopes(mongoose, owner, ['conversation-one']), + ]); + expect(await storage.read(owner, 'resp_one')).toMatchObject({ responseId: 'resp_one' }); + expect(await storage.reserve(reservation({ conversationId: 'fresh-conversation' }))).toEqual({ + ok: false, + code: 'response_unavailable', + }); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + })); + +it('does not close another tenant or owner and retains only hashed scope metadata', () => + scoped(async () => { + await closeResponseScopes(mongoose, { ...owner, tenantId: 'other-tenant' }); + await closeResponseScopes(mongoose, { ...owner, userId: 'other-owner' }); + expect((await storage.reserve(reservation())).ok).toBe(true); + const anchors = JSON.stringify(await mongoose.models.ResponseScope.find({}).lean()); + expect(anchors).not.toContain(owner.userId); + expect(anchors).not.toContain(owner.tenantId); + expect(anchors).not.toContain('Hello'); + })); + +it.each(['owner', 'conversation', 'retention'])( + 'waits for admitted insertion before %s erasure sweeps records', + (kind) => + scoped(async () => { + let entered!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const collection = mongoose.models.StoredResponse.collection; + const insert = collection.insertOne.bind(collection); + const paused = jest.spyOn(collection, 'insertOne').mockImplementation(async (...args) => { + entered(); + await gate; + return insert(...args); + }); + const pending = storage.reserve(reservation()); + await waitForPause(started, pending); + let erased = false; + const intent = + kind === 'retention' + ? createResponseRetention(mongoose)(owner, 'conversation-one', new Date(Date.now() - 1)) + : closeResponseScopes( + mongoose, + owner, + kind === 'conversation' ? ['conversation-one'] : undefined, + ); + const erasure = intent.then(async () => { + await mongoose.models.StoredResponse.deleteMany({ userId: owner.userId }); + erased = true; + }); + const drained = Promise.allSettled([pending, erasure]); + try { + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(erased).toBe(false); + } finally { + release(); + paused.mockRestore(); + await drained; + } + expect((await pending).ok).toBe(true); + await erasure; + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + expect(await storage.reserve(reservation())).toEqual({ + ok: false, + code: 'response_unavailable', + }); + }), +); + +it.each(['owner_deleted', 'conversation_deleted', 'expired'] as const)( + 'keeps %s cleanup authority after private row removal without inventing a deadline', + (retirement) => + scoped(async () => { + await storage.reserve(reservation()); + await mongoose.models.StoredResponse.updateOne( + { responseId: 'resp_one' }, + { + $set: { + generationCreatedAt: 42, + pendingTerminal: { + generationCreatedAt: 42, + providerExecutionId: 'initial-provider', + status: 'failed', + responseJSON: '{"private":"answer"}', + checkpointNamespace: 'checkpoint-one', + checkpointConfigJSON: '{"saver":"pinned"}', + }, + }, + }, + ); + if (retirement === 'expired') + await createResponseRetention(mongoose)( + owner, + 'conversation-one', + new Date(Date.now() - 1000), + ); + else + await closeResponseScopes( + mongoose, + owner, + retirement === 'conversation_deleted' ? ['conversation-one'] : undefined, + ); + const identity = await storage.readCleanupIdentity(owner, 'resp_one'); + expect(identity).toMatchObject({ + retirement, + generationCreatedAt: 42, + checkpointNamespace: 'checkpoint-one', + checkpointConfigJSON: '{"saver":"pinned"}', + requiresCheckpointCleanup: true, + }); + expect(identity).not.toHaveProperty('requestJSON'); + expect(identity).not.toHaveProperty('responseJSON'); + expect(identity).not.toHaveProperty('pendingTerminal'); + if (retirement !== 'expired') expect(identity?.expiredAt).toBeUndefined(); + await mongoose.models.StoredResponse.deleteOne({ responseId: 'resp_one' }); + expect(await storage.readCleanupIdentity(owner, 'resp_one', identity!)).toEqual(identity); + expect( + await storage.readCleanupIdentity(owner, 'resp_one', { + ...identity!, + userId: 'another-owner', + }), + ).toBeNull(); + expect( + await storage.readCleanupIdentity(owner, 'resp_one', { + ...identity!, + tenantId: 'another-tenant', + }), + ).toBeNull(); + expect( + await storage.readCleanupIdentity(owner, 'resp_one', { + ...identity!, + responseId: 'another-response', + }), + ).toBeNull(); + }), +); + +it('requires checkpoint cleanup for an initial staged terminal without a review or retained identity', () => + scoped(async () => { + await storage.reserve(reservation()); + const running = await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt: 42, + responseJSON: '{}', + }); + expect(running.ok).toBe(true); + if (!running.ok) return; + const staged = await storage.stageTerminal(owner, 'resp_one', running.record.revision, { + generationCreatedAt: 42, + providerExecutionId: 'initial-provider', + status: 'failed', + responseJSON: '{}', + checkpointNamespace: 'checkpoint-one', + checkpointConfigJSON: '{"saver":"initial"}', + }); + expect(staged.ok).toBe(true); + expect(await storage.readCleanupIdentity(owner, 'resp_one')).toMatchObject({ + checkpointNamespace: 'checkpoint-one', + checkpointConfigJSON: '{"saver":"initial"}', + requiresCheckpointCleanup: true, + }); + })); + +it.each([ + { userId: 'another-owner' }, + { tenantId: 'another-tenant' }, + { responseId: 'another-response' }, + { conversationId: 'another-conversation' }, + { agentId: 'another-agent' }, + { generationCreatedAt: 43 }, +])('never mixes retained checkpoint metadata from a different identity: %p', (changed) => + scoped(async () => { + await storage.reserve(reservation()); + await mongoose.models.StoredResponse.updateOne( + { responseId: 'resp_one' }, + { + $set: { generationCreatedAt: 42 }, + }, + ); + const identity = await storage.readCleanupIdentity(owner, 'resp_one'); + expect( + await storage.readCleanupIdentity(owner, 'resp_one', { + ...identity!, + ...changed, + checkpointNamespace: 'wrong-checkpoint', + checkpointConfigJSON: 'wrong-config', + requiresCheckpointCleanup: true, + }), + ).toMatchObject({ + checkpointNamespace: undefined, + checkpointConfigJSON: undefined, + requiresCheckpointCleanup: false, + }); + }), +); + +it('rejects a missing row with no retirement proof even when retained data claims expiry', () => + scoped(async () => { + await storage.reserve(reservation()); + const retained = await storage.readCleanupIdentity(owner, 'resp_one'); + await mongoose.models.StoredResponse.deleteOne({ responseId: 'resp_one' }); + expect( + await storage.readCleanupIdentity(owner, 'resp_one', { + ...retained!, + expiredAt: new Date(Date.now() - 1), + retirement: 'expired', + }), + ).toBeNull(); + await closeResponseScopes(mongoose, { ...owner, userId: 'another-owner' }); + await closeResponseScopes(mongoose, owner, ['another-conversation']); + expect(await storage.readCleanupIdentity(owner, 'resp_one', retained!)).toBeNull(); + })); + +it('recovers and erases all legacy partitions inside an ordinary tenant request', async () => { + createModels(mongoose); + const tenants = ['tenant-a', 'tenant-b', undefined, '', null]; + await runAsSystem(async () => { + for (const [index, tenantId] of tenants.entries()) { + const responseId = `legacy-${index}`; + // Distinct request keys let all tenantless legacy spellings coexist. + await storage.reserve( + reservation({ responseId, requestKey: responseId, tenantId: tenantId || undefined }), + ); + if (tenantId !== undefined) + await mongoose.models.StoredResponse.collection.updateOne( + { responseId }, + { $set: { tenantId } }, + ); + } + await storage.reserve(reservation({ userId: 'another-owner', responseId: 'unrelated-owner' })); + await storage.reserve( + reservation({ + conversationId: 'another-conversation', + responseId: 'unrelated-conversation', + requestKey: 'unrelated-conversation', + }), + ); + }); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + await scoped(async () => { + await methods.deleteConvos(owner.userId, { conversationId: 'conversation-one' }); + await methods.deleteConvos( + owner.userId, + { conversationId: 'conversation-one' }, + { allowEmpty: true }, + ); + }); + await runAsSystem(async () => { + const remaining = await mongoose.models.StoredResponse.find({}).select('responseId').lean(); + expect(remaining.map((row) => row.responseId).sort()).toEqual([ + 'unrelated-conversation', + 'unrelated-owner', + ]); + for (const tenantId of ['tenant-a', 'tenant-b', undefined]) + expect( + await storage.reserve(reservation({ tenantId, requestKey: `late-${tenantId}` })), + ).toEqual({ ok: false, code: 'response_unavailable' }); + }); +}); + +it('keeps incomplete staged checkpoint bindings retryable instead of mixing older settings', () => + scoped(async () => { + await storage.reserve(reservation()); + await mongoose.models.StoredResponse.updateOne( + { responseId: 'resp_one' }, + { + $set: { + generationCreatedAt: 42, + review: { + actionId: 'old-review', + providerExecutionId: 'old-provider', + expiresAt: Date.now() + 1000, + partialResponseJSON: '{}', + checkpointConfigJSON: 'old-config', + }, + pendingTerminal: { + generationCreatedAt: 42, + providerExecutionId: 'new-provider', + status: 'failed', + responseJSON: '{}', + checkpointNamespace: 'new-checkpoint', + }, + }, + }, + ); + const retained = { + ...owner, + responseId: 'resp_one', + agentId: 'agent-one', + conversationId: 'conversation-one', + generationCreatedAt: 42, + checkpointNamespace: 'old-checkpoint', + checkpointConfigJSON: 'retained-config', + }; + expect(await storage.readCleanupIdentity(owner, 'resp_one', retained)).toMatchObject({ + checkpointNamespace: 'new-checkpoint', + checkpointConfigJSON: undefined, + requiresCheckpointCleanup: true, + }); + })); + +it('propagates cleanup lookup outages instead of treating retained metadata as authority', () => + scoped(async () => { + await storage.reserve(reservation()); + const retained = await storage.readCleanupIdentity(owner, 'resp_one'); + jest + .spyOn(mongoose.models.StoredResponse.collection, 'findOne') + .mockRejectedValueOnce(new Error('database unavailable')); + await expect(storage.readCleanupIdentity(owner, 'resp_one', retained!)).rejects.toThrow( + 'database unavailable', + ); + })); + +it.each(['session', 'commit'] as const)( + 'retries reservation after a transient capability %s failure', + (failure) => + scoped(async () => { + const fresh = createResponseStorage(mongoose); + if (failure === 'session') + jest + .spyOn(mongoose, 'startSession') + .mockRejectedValueOnce(new Error('transient probe outage')); + else { + const session = await mongoose.startSession(); + jest + .spyOn(session, 'commitTransaction') + .mockRejectedValueOnce(new Error('transient probe outage')); + jest.spyOn(mongoose, 'startSession').mockResolvedValueOnce(session); + } + await expect(fresh.reserve(reservation())).rejects.toThrow('transient probe outage'); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + expect(await mongoose.models.ResponseScope.countDocuments({})).toBe(0); + expect(await fresh.reserve(reservation())).toMatchObject({ ok: true, replayed: false }); + }), +); + +it('keeps response-bound conversation identity immutable without cancelling its work', () => + scoped(async () => { + createModels(mongoose); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + await methods.saveConvo( + { userId: owner.userId }, + { conversationId: 'immutable-source', endpoint: 'agents' }, + ); + await storage.reserve(reservation({ conversationId: 'immutable-source' })); + expect( + await methods.saveConvo( + { userId: owner.userId }, + { + conversationId: 'immutable-source', + newConversationId: 'replacement', + endpoint: 'agents', + }, + ), + ).toEqual({ message: 'Conversation ID cannot be replaced after persistence.' }); + expect( + await mongoose.models.Conversation.exists({ + user: owner.userId, + conversationId: 'immutable-source', + }), + ).not.toBeNull(); + expect( + await mongoose.models.Conversation.exists({ + user: owner.userId, + conversationId: 'replacement', + }), + ).toBeNull(); + expect(await storage.read(owner, 'resp_one')).not.toHaveProperty('cancellationRequestedAt'); + expect( + await storage.reserve( + reservation({ + responseId: 'another', + requestKey: 'another', + conversationId: 'immutable-source', + }), + ), + ).toMatchObject({ ok: true }); + })); + +it('fences an empty old identity and propagates retention to its replacement before admitting work', () => + scoped(async () => { + createModels(mongoose); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + const expiredAt = new Date(Date.now() + 60_000); + expect( + await methods.saveConvo( + { userId: owner.userId, expiredAt }, + { + conversationId: 'initial-source', + newConversationId: 'replacement', + endpoint: 'agents', + }, + ), + ).toMatchObject({ conversationId: 'replacement' }); + expect(await storage.reserve(reservation({ conversationId: 'initial-source' }))).toEqual({ + ok: false, + code: 'response_unavailable', + }); + expect(await storage.reserve(reservation({ conversationId: 'replacement' }))).toMatchObject({ + ok: true, + record: { expiredAt }, + }); + await methods.deleteConvos(owner.userId, { conversationId: 'replacement' }); + expect(await storage.read(owner, 'resp_one')).toBeNull(); + })); + +it('serializes ID replacement with a reservation already inserting private data', () => + scoped(async () => { + createModels(mongoose); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + let entered!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const collection = mongoose.models.StoredResponse.collection; + const insert = collection.insertOne.bind(collection); + const paused = jest.spyOn(collection, 'insertOne').mockImplementation(async (...args) => { + entered(); + await gate; + return insert(...args); + }); + const pending = storage.reserve(reservation({ conversationId: 'racing-source' })); + await waitForPause(started, pending); + let settled = false; + const replacement = methods + .saveConvo( + { userId: owner.userId }, + { + conversationId: 'racing-source', + newConversationId: 'replacement', + endpoint: 'agents', + }, + ) + .then((value) => { + settled = true; + return value; + }); + const drained = Promise.allSettled([pending, replacement]); + try { + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(settled).toBe(false); + } finally { + release(); + paused.mockRestore(); + await drained; + } + expect(await pending).toMatchObject({ ok: true }); + expect(await replacement).toEqual({ + message: 'Conversation ID cannot be replaced after persistence.', + }); + expect(await storage.read(owner, 'resp_one')).toMatchObject({ + conversationId: 'racing-source', + }); + expect( + await mongoose.models.Conversation.exists({ conversationId: 'racing-source' }), + ).toBeNull(); + })); + +it.each(['transient_read', 'missing_canary', 'failed_canary_create'] as const)( + 'classifies capability probe errors without caching an outage: %s', + (kind) => + scoped(async () => { + const db = mongoose.connection.db!; + const probe = db.collection('__transaction_test__'); + const collection = db.collection.bind(db); + jest + .spyOn(db, 'collection') + .mockImplementation((name, options) => + name === '__transaction_test__' ? probe : collection(name, options), + ); + const failure = + kind === 'transient_read' + ? new Error('transient database read outage') + : new mongoose.mongo.MongoServerError({ + code: 303, + message: 'Feature not supported: non-existent collection in transaction', + }); + const read = jest.spyOn(probe, 'findOne').mockRejectedValueOnce(failure); + if (kind === 'failed_canary_create') + jest + .spyOn(db, 'createCollection') + .mockRejectedValueOnce(new Error('canary creation denied')); + const fresh = createResponseStorage(mongoose); + if (kind === 'missing_canary') { + expect(await fresh.reserve(reservation())).toMatchObject({ ok: true }); + expect(read).toHaveBeenCalledTimes(2); + } else { + await expect(fresh.reserve(reservation())).rejects.toThrow( + kind === 'transient_read' ? 'transient database read outage' : 'canary creation denied', + ); + expect(await fresh.reserve(reservation())).toMatchObject({ ok: true }); + } + }), +); + +it('never renames a source conversation inserted after initial assignment checked absence', () => + scoped(async () => { + createModels(mongoose); + const sourceId = 'late-initial-source'; + const collection = mongoose.models.Conversation.collection; + const find = collection.findOne.bind(collection); + let inserted = false; + jest.spyOn(collection, 'findOne').mockImplementation(async (...args) => { + const found = await find(...args); + if (!inserted && args[0]?.conversationId === sourceId && found == null) { + inserted = true; + await collection.insertOne({ + user: owner.userId, + tenantId: owner.tenantId, + conversationId: sourceId, + endpoint: 'agents', + title: 'Concurrent source', + }); + } + return found; + }); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + expect( + await methods.saveConvo( + { userId: owner.userId }, + { + conversationId: sourceId, + newConversationId: 'initial-destination', + endpoint: 'agents', + }, + ), + ).toMatchObject({ conversationId: 'initial-destination' }); + expect( + await mongoose.models.Conversation.findOne({ conversationId: sourceId }).lean(), + ).toMatchObject({ title: 'Concurrent source' }); + })); + +it('retries a failed initial ID assignment without reopening its unused source scope', () => + scoped(async () => { + createModels(mongoose); + const methods = createConversationMethods(mongoose, { + getMessages: jest.fn(async () => []), + deleteMessages: jest.fn(async () => ({ acknowledged: true, deletedCount: 0 })), + }); + jest + .spyOn(mongoose.models.Conversation.collection, 'findOneAndUpdate') + .mockRejectedValueOnce(new Error('parent write unavailable')); + const input = { + conversationId: 'failed-initial-source', + newConversationId: 'retry-initial-destination', + endpoint: 'agents', + }; + expect(await methods.saveConvo({ userId: owner.userId }, input)).toEqual({ + message: 'Error saving conversation', + }); + expect(await storage.reserve(reservation({ conversationId: input.conversationId }))).toEqual({ + ok: false, + code: 'response_unavailable', + }); + expect(await methods.saveConvo({ userId: owner.userId }, input)).toMatchObject({ + conversationId: input.newConversationId, + }); + expect( + await storage.reserve(reservation({ conversationId: input.newConversationId })), + ).toMatchObject({ ok: true }); + })); diff --git a/packages/data-schemas/src/methods/response.ts b/packages/data-schemas/src/methods/response.ts new file mode 100644 index 0000000000..d50bc783a3 --- /dev/null +++ b/packages/data-schemas/src/methods/response.ts @@ -0,0 +1,691 @@ +import type { + ResponseOwner, + StoredResponseRecord, + StoredResponseStatus, + ResponseReviewCheckpoint, + ResponseResumeIntent, + ResponseTerminalReceipt, +} from '~/models/response'; +import { + responseScopeKeys, + responseOwnerFilter, + isResponseScopeClosed, + initializeResponseScopes, +} from './responseScope'; +import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext'; +import { createResponseScopeModel } from '~/models/responseScope'; +import { createStoredResponseModel } from '~/models/response'; +import { getTransactionSupport } from '~/utils/transactions'; +import { activeExpirationFilter } from '~/utils/retention'; +import { createIndexesWithRetry } from '~/utils/retry'; + +export type { + ResponseOwner, + StoredResponseRecord, + StoredResponseStatus, + ResponseReviewCheckpoint, + ResponseResumeIntent, + ResponseTerminalReceipt, +} from '~/models/response'; + +export interface ResponseReservation extends ResponseOwner { + expiredAt?: Date; + isTemporary?: boolean; + responseId: string; + requestKey: string; + fingerprint: string; + agentId: string; + sourceKeyId?: string; + requestJSON: string; + conversationId: string; + parentMessageId: string; +} + +export type ResponseReservationResult = + | { ok: true; record: StoredResponseRecord; replayed: boolean } + | { ok: false; code: 'idempotency_conflict' | 'response_unavailable' }; + +export type ResponseMutationResult = + | { ok: true; record: StoredResponseRecord } + | { ok: false; code: 'response_conflict' }; + +/** Internal drain identity only; expired requests, answers and decisions are never returned. */ +export type ResponseCleanupIdentity = Pick< + StoredResponseRecord, + | 'userId' + | 'tenantId' + | 'responseId' + | 'agentId' + | 'conversationId' + | 'generationCreatedAt' + | 'expiredAt' +> & { + /** Cleanup authority only; never permission to read output or execute a provider. */ + retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired'; + checkpointNamespace?: string; + checkpointConfigJSON?: string; + requiresCheckpointCleanup?: boolean; +}; + +export interface ResponseResumeReservation + extends Omit { + generationCreatedAt: number; +} + +/** Replays are receipts, not permission to enqueue or execute terminal/cancelled runs. */ +export type ResponseResumeResult = + | { ok: true; record: StoredResponseRecord; replayed: boolean } + | { ok: false; code: 'resume_conflict' }; + +export interface ResponseStorage { + stageTerminal( + owner: ResponseOwner, + responseId: string, + revision: number, + receipt: ResponseTerminalReceipt, + ): Promise; + finishTerminal( + owner: ResponseOwner, + responseId: string, + revision: number, + receipt: ResponseTerminalReceipt, + ): Promise; + publishReview( + owner: ResponseOwner, + responseId: string, + revision: number, + generationCreatedAt: number, + review: ResponseReviewCheckpoint, + responseJSON: string, + ): Promise; + reserveResume( + owner: ResponseOwner, + responseId: string, + input: ResponseResumeReservation, + now: number, + ): Promise; + admitResume( + owner: ResponseOwner, + responseId: string, + intentId: string, + generationCreatedAt: number, + providerExecutionId: string, + responseJSON: string, + now: number, + ): Promise; + reserve(input: ResponseReservation): Promise; + read(owner: ResponseOwner, responseId: string): Promise; + readCleanupIdentity( + owner: ResponseOwner, + responseId: string, + retained?: ResponseCleanupIdentity, + ): Promise; + attachDelivery( + owner: ResponseOwner, + responseId: string, + deliveryKey: string, + ): Promise; + publish( + owner: ResponseOwner, + responseId: string, + revision: number, + update: { status: StoredResponseStatus; responseJSON: string; generationCreatedAt?: number }, + ): Promise; + requestCancellation( + owner: ResponseOwner, + responseId: string, + now: number, + ): Promise; +} + +const active = ['queued', 'in_progress', 'requires_action']; + +function sameTerminal(a: ResponseTerminalReceipt, b: ResponseTerminalReceipt): boolean { + return ( + a.generationCreatedAt === b.generationCreatedAt && + a.providerExecutionId === b.providerExecutionId && + a.ownerId === b.ownerId && + a.status === b.status && + a.responseJSON === b.responseJSON && + a.checkpointNamespace === b.checkpointNamespace && + a.checkpointConfigJSON === b.checkpointConfigJSON + ); +} + +/** Authorization precedes this storage boundary; every read/write remains owner- and tenant-scoped. */ +export function createResponseStorage(mongoose: typeof import('mongoose')): ResponseStorage { + const model = createStoredResponseModel(mongoose); + let indexPromise: Promise | undefined; + let transactionSupport: boolean | null = null; + function ensureIndexes() { + indexPromise ??= createIndexesWithRetry(model) + .then(() => undefined) + .catch((error: unknown) => { + indexPromise = undefined; + throw error; + }); + return indexPromise; + } + function ownerScope(owner: ResponseOwner) { + if (!owner.userId || owner.tenantId === '__SYSTEM__') + throw new TypeError('Invalid response owner'); + const tenantId = getTenantId() || undefined; + if ( + tenantId != null && + tenantId !== SYSTEM_TENANT_ID && + tenantId !== (owner.tenantId || undefined) + ) + throw new TypeError('Response tenant does not match authenticated scope'); + return responseOwnerFilter(owner); + } + const scope = (owner: ResponseOwner) => ({ + ...ownerScope(owner), + $and: [activeExpirationFilter()], + }); + return { + async stageTerminal(owner, responseId, revision, receipt) { + const pendingTerminal: ResponseTerminalReceipt = { + generationCreatedAt: receipt.generationCreatedAt, + providerExecutionId: receipt.providerExecutionId, + ...(receipt.ownerId == null ? {} : { ownerId: receipt.ownerId }), + status: receipt.status, + responseJSON: receipt.responseJSON, + ...(receipt.checkpointNamespace == null + ? {} + : { checkpointNamespace: receipt.checkpointNamespace }), + ...(receipt.checkpointConfigJSON == null + ? {} + : { checkpointConfigJSON: receipt.checkpointConfigJSON }), + }; + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + revision, + generationCreatedAt: receipt.generationCreatedAt, + status: { $in: active }, + pendingTerminal: { $exists: false }, + $or: [ + { + 'resume.state': 'admitted', + 'resume.providerExecutionId': receipt.providerExecutionId, + }, + { + 'resume.state': { $ne: 'admitted' }, + 'review.providerExecutionId': receipt.providerExecutionId, + }, + { review: { $exists: false }, resume: { $exists: false } }, + ], + }, + { $set: { pendingTerminal }, $inc: { revision: 1 } }, + { new: true, runValidators: true }, + ) + .lean(); + if (record) return { ok: true, record }; + const existing = await model.findOne({ ...scope(owner), responseId }).lean(); + if ( + existing?.generationCreatedAt === receipt.generationCreatedAt && + existing.pendingTerminal && + sameTerminal(existing.pendingTerminal, pendingTerminal) + ) + return { ok: true, record: existing }; + return { ok: false, code: 'response_conflict' }; + }, + async finishTerminal(owner, responseId, revision, receipt) { + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + revision, + generationCreatedAt: receipt.generationCreatedAt, + status: { $in: active }, + 'pendingTerminal.generationCreatedAt': receipt.generationCreatedAt, + 'pendingTerminal.providerExecutionId': receipt.providerExecutionId, + 'pendingTerminal.ownerId': receipt.ownerId ?? { $exists: false }, + 'pendingTerminal.status': receipt.status, + 'pendingTerminal.responseJSON': receipt.responseJSON, + 'pendingTerminal.checkpointNamespace': receipt.checkpointNamespace ?? { + $exists: false, + }, + 'pendingTerminal.checkpointConfigJSON': receipt.checkpointConfigJSON ?? { + $exists: false, + }, + }, + { + $set: { status: receipt.status, responseJSON: receipt.responseJSON }, + $unset: { pendingTerminal: 1 }, + $inc: { revision: 1 }, + }, + { new: true }, + ) + .lean(); + return record ? { ok: true, record } : { ok: false, code: 'response_conflict' }; + }, + async publishReview(owner, responseId, revision, generationCreatedAt, review, responseJSON) { + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + revision, + generationCreatedAt, + status: { $in: ['in_progress', 'requires_action'] }, + pendingTerminal: { $exists: false }, + cancellationRequestedAt: { $exists: false }, + $or: [ + { review: { $exists: false }, resume: { $exists: false } }, + { + 'review.actionId': review.actionId, + 'review.providerExecutionId': review.providerExecutionId, + resume: { $exists: false }, + }, + { + 'resume.state': 'admitted', + 'resume.providerExecutionId': review.providerExecutionId, + }, + ], + }, + { + $set: { + status: 'requires_action', + review: { + actionId: review.actionId, + providerExecutionId: review.providerExecutionId, + expiresAt: review.expiresAt, + partialResponseJSON: review.partialResponseJSON, + ...(review.visibleOutputTokens == null + ? {} + : { visibleOutputTokens: review.visibleOutputTokens }), + ...(review.pendingActionJSON == null + ? {} + : { pendingActionJSON: review.pendingActionJSON }), + ...(review.checkpointConfigJSON == null + ? {} + : { checkpointConfigJSON: review.checkpointConfigJSON }), + ...(review.checkpointId == null ? {} : { checkpointId: review.checkpointId }), + ...(review.checkpointNs == null ? {} : { checkpointNs: review.checkpointNs }), + }, + responseJSON, + }, + $unset: { resume: 1 }, + $inc: { revision: 1 }, + }, + { new: true, runValidators: true }, + ) + .lean(); + return record ? { ok: true, record } : { ok: false, code: 'response_conflict' }; + }, + async reserveResume(owner, responseId, input, now) { + if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume reservation time'); + if (input.providerExecutionId === input.previousProviderExecutionId) + throw new TypeError('Resume requires a new provider execution identity'); + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + generationCreatedAt: input.generationCreatedAt, + status: 'requires_action', + pendingTerminal: { $exists: false }, + cancellationRequestedAt: { $exists: false }, + resume: { $exists: false }, + 'review.actionId': input.actionId, + 'review.providerExecutionId': input.previousProviderExecutionId, + 'review.expiresAt': { $gt: now }, + }, + { + $set: { + resume: { + intentId: input.intentId, + actionId: input.actionId, + previousProviderExecutionId: input.previousProviderExecutionId, + providerExecutionId: input.providerExecutionId, + fingerprint: input.fingerprint, + resolutionJSON: input.resolutionJSON, + ...(input.decisionJSON == null ? {} : { decisionJSON: input.decisionJSON }), + state: 'reserved', + reservedAt: now, + }, + }, + $inc: { revision: 1 }, + }, + { new: true, runValidators: true }, + ) + .lean(); + if (record) return { ok: true, record, replayed: false }; + const existing = await model.findOne({ ...scope(owner), responseId }).lean(); + const intent = existing?.resume; + if ( + existing?.generationCreatedAt === input.generationCreatedAt && + intent && + intent.actionId === input.actionId && + intent.previousProviderExecutionId === input.previousProviderExecutionId && + intent.fingerprint === input.fingerprint && + intent.resolutionJSON === input.resolutionJSON && + intent.decisionJSON === input.decisionJSON + ) + return { ok: true, record: existing, replayed: true }; + return { ok: false, code: 'resume_conflict' }; + }, + async admitResume( + owner, + responseId, + intentId, + generationCreatedAt, + providerExecutionId, + responseJSON, + now, + ) { + if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid resume admission time'); + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + generationCreatedAt, + status: 'requires_action', + pendingTerminal: { $exists: false }, + cancellationRequestedAt: { $exists: false }, + 'resume.intentId': intentId, + 'resume.providerExecutionId': providerExecutionId, + 'resume.state': 'reserved', + }, + { + $set: { + status: 'in_progress', + responseJSON, + 'resume.state': 'admitted', + 'resume.admittedAt': now, + }, + $inc: { revision: 1 }, + }, + { new: true }, + ) + .lean(); + if (record) return { ok: true, record }; + const existing = await model.findOne({ ...scope(owner), responseId }).lean(); + if ( + existing?.generationCreatedAt === generationCreatedAt && + existing.resume?.intentId === intentId && + existing.resume.providerExecutionId === providerExecutionId && + existing.resume.state === 'admitted' + ) + return { ok: true, record: existing }; + return { ok: false, code: 'response_conflict' }; + }, + async reserve(input) { + if ( + input.expiredAt != null && + (!(input.expiredAt instanceof Date) || !Number.isFinite(input.expiredAt.getTime())) + ) + throw new TypeError('Invalid response expiration'); + const filter = { ...scope(input), requestKey: input.requestKey }; + transactionSupport = await getTransactionSupport(mongoose, transactionSupport); + if (!transactionSupport) return { ok: false, code: 'response_unavailable' }; + await ensureIndexes(); + const prior = await model.findOne(filter).select('conversationId').lean(); + const keys = responseScopeKeys(input, prior?.conversationId ?? input.conversationId); + await initializeResponseScopes(mongoose, keys); + const session = await mongoose.startSession(); + try { + let result: ResponseReservationResult | undefined; + await session.withTransaction(async () => { + // A write, not a snapshot read: erasure waits for admitted insertions to commit. + const scopes = createResponseScopeModel(mongoose); + const touched = await scopes.updateOne( + { _id: keys[0], closed: false }, + { $inc: { revision: 1 } }, + { session }, + ); + if (touched.matchedCount !== 1) { + result = { ok: false, code: 'response_unavailable' }; + return; + } + const existing = await model.findOne(filter).session(session).lean(); + const conversationKey = responseScopeKeys( + input, + existing?.conversationId ?? input.conversationId, + )[1]; + const conversation = await scopes + .findOneAndUpdate( + { _id: conversationKey, closed: false }, + { + $inc: { revision: 1 }, + ...(input.expiredAt ? { $min: { expiredAt: input.expiredAt } } : {}), + }, + { session, new: true }, + ) + .lean(); + if (!conversation) { + result = { ok: false, code: 'response_unavailable' }; + return; + } + if (conversation.expiredAt) { + await model.updateMany( + { + ...responseOwnerFilter(input), + conversationId: existing?.conversationId ?? input.conversationId, + $or: [{ expiredAt: null }, { expiredAt: { $gt: conversation.expiredAt } }], + }, + { $set: { expiredAt: conversation.expiredAt } }, + { session }, + ); + if (existing && (!existing.expiredAt || conversation.expiredAt < existing.expiredAt)) + existing.expiredAt = conversation.expiredAt; + if (conversation.expiredAt.getTime() <= Date.now()) { + result = { ok: false, code: 'response_unavailable' }; + return; + } + } + if (existing) { + result = + existing.fingerprint === input.fingerprint + ? { ok: true, record: existing, replayed: true } + : { ok: false, code: 'idempotency_conflict' }; + return; + } + const [created] = await model.create( + [ + { + ...input, + ...(conversation.expiredAt ? { expiredAt: conversation.expiredAt } : {}), + status: 'queued', + revision: 0, + }, + ], + { + session, + }, + ); + result = { ok: true, record: created.toObject(), replayed: false }; + }); + if (!result) throw new Error('Response reservation did not commit'); + return result; + } catch (error) { + if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 11000) + throw error; + const existing = await model.findOne(filter).lean(); + if (!existing) { + if (await model.exists({ ...responseOwnerFilter(input), requestKey: input.requestKey })) + return { ok: false, code: 'response_unavailable' }; + throw error; + } + if (await isResponseScopeClosed(mongoose, input, existing.conversationId)) + return { ok: false, code: 'response_unavailable' }; + return existing.fingerprint === input.fingerprint + ? { ok: true, record: existing, replayed: true } + : { ok: false, code: 'idempotency_conflict' }; + } finally { + await session.endSession(); + } + }, + async read(owner, responseId) { + return model.findOne({ ...scope(owner), responseId }).lean(); + }, + async readCleanupIdentity(owner, responseId, retained) { + const record = await model + .findOne({ ...ownerScope(owner), responseId }) + .select( + 'userId tenantId responseId agentId conversationId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId', + ) + .lean(); + // Retained data comes only from an internal job/receipt, never request input. + // Validate it even when the row exists before using any fallback checkpoint fields. + const matchingRetained = + retained && + retained.responseId === responseId && + retained.userId === owner.userId && + (retained.tenantId || undefined) === (owner.tenantId || undefined) && + (!record || + (retained.conversationId === record.conversationId && + retained.agentId === record.agentId && + retained.generationCreatedAt === record.generationCreatedAt)) + ? retained + : undefined; + const source = record ?? matchingRetained; + if (!source) return null; + const now = new Date(); + const [ownerKey, conversationKey] = responseScopeKeys(owner, source.conversationId); + const anchors = await createResponseScopeModel(mongoose) + .find({ _id: { $in: [ownerKey, conversationKey] } }) + .select('closed expiredAt') + .lean(); + const ownerAnchor = anchors.find((anchor) => anchor._id === ownerKey); + const conversationAnchor = anchors.find((anchor) => anchor._id === conversationKey); + const expiredAt = + conversationAnchor?.expiredAt && + (!source.expiredAt || conversationAnchor.expiredAt < source.expiredAt) + ? conversationAnchor.expiredAt + : source.expiredAt; + let retirement: ResponseCleanupIdentity['retirement']; + if (ownerAnchor?.closed) retirement = 'owner_deleted'; + else if (conversationAnchor?.closed) retirement = 'conversation_deleted'; + else if (expiredAt && expiredAt <= now) retirement = 'expired'; + // A retained deadline alone cannot prove why an authoritative row disappeared. + if ( + !record && + !ownerAnchor?.closed && + !conversationAnchor?.closed && + !(conversationAnchor?.expiredAt && conversationAnchor.expiredAt <= now) + ) + return null; + // A staged checkpoint is one binding: never fill an incomplete terminal + // binding with settings from a prior review or retained provider segment. + const terminalCheckpoint = record?.pendingTerminal; + const hasTerminalCheckpoint = + terminalCheckpoint?.checkpointNamespace != null || + terminalCheckpoint?.checkpointConfigJSON != null; + const checkpointNamespace = hasTerminalCheckpoint + ? terminalCheckpoint?.checkpointNamespace + : matchingRetained?.checkpointNamespace; + const checkpointConfigJSON = hasTerminalCheckpoint + ? terminalCheckpoint?.checkpointConfigJSON + : (record?.review?.checkpointConfigJSON ?? matchingRetained?.checkpointConfigJSON); + const { + userId, + tenantId, + responseId: id, + agentId, + conversationId, + generationCreatedAt, + } = source; + return { + userId, + tenantId, + responseId: id, + agentId, + conversationId, + generationCreatedAt, + expiredAt, + retirement, + checkpointNamespace, + checkpointConfigJSON, + requiresCheckpointCleanup: Boolean( + record?.review || + checkpointNamespace || + checkpointConfigJSON || + matchingRetained?.requiresCheckpointCleanup, + ), + }; + }, + async attachDelivery(owner, responseId, deliveryKey) { + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + status: { $in: active }, + $or: [{ deliveryKey: { $exists: false } }, { deliveryKey }], + }, + { $set: { deliveryKey } }, + { new: true }, + ) + .lean(); + return record ? { ok: true, record } : { ok: false, code: 'response_conflict' }; + }, + async publish(owner, responseId, revision, update) { + if ( + update.generationCreatedAt != null && + (!Number.isSafeInteger(update.generationCreatedAt) || update.generationCreatedAt <= 0) + ) + throw new TypeError('Invalid response generation identity'); + let predecessors = active; + if (update.status === 'queued') predecessors = ['queued']; + if (update.status === 'in_progress') predecessors = ['queued', 'in_progress']; + const record = await model + .findOneAndUpdate( + { + ...scope(owner), + responseId, + revision, + status: { $in: predecessors }, + pendingTerminal: { $exists: false }, + ...(update.status === 'in_progress' + ? { + $and: [ + activeExpirationFilter(), + { + $or: [ + { status: 'in_progress' }, + { cancellationRequestedAt: { $exists: false } }, + ], + }, + ...(update.generationCreatedAt == null + ? [{ generationCreatedAt: { $gt: 0 } }] + : []), + ], + } + : {}), + ...(update.generationCreatedAt == null + ? {} + : { + $or: [ + { generationCreatedAt: { $exists: false } }, + { generationCreatedAt: update.generationCreatedAt }, + ], + }), + }, + { $set: update, $inc: { revision: 1 } }, + { new: true }, + ) + .lean(); + return record ? { ok: true, record } : { ok: false, code: 'response_conflict' }; + }, + async requestCancellation(owner, responseId, now) { + if (!Number.isFinite(now) || now <= 0) throw new TypeError('Invalid cancellation time'); + await model.updateOne( + { + ...scope(owner), + responseId, + status: { $in: active }, + cancellationRequestedAt: { $exists: false }, + pendingTerminal: { $exists: false }, + }, + { $set: { cancellationRequestedAt: now } }, + ); + return model.findOne({ ...scope(owner), responseId }).lean(); + }, + }; +} diff --git a/packages/data-schemas/src/methods/responseScope.ts b/packages/data-schemas/src/methods/responseScope.ts new file mode 100644 index 0000000000..1b7474ae3b --- /dev/null +++ b/packages/data-schemas/src/methods/responseScope.ts @@ -0,0 +1,224 @@ +import { createHash } from 'node:crypto'; +import type { ResponseOwner } from '~/models/response'; +import { createResponseScopeModel } from '~/models/responseScope'; +import { createStoredResponseModel } from '~/models/response'; +import { getTransactionSupport } from '~/utils/transactions'; +import { createIndexesWithRetry } from '~/utils/retry'; +import { runAsSystem } from '~/config/tenantContext'; + +/** Response-bound conversation IDs are immutable: jobs and checkpoints retain that identity. + * An empty source scope is closed before replacement so delayed admission cannot orphan data. */ +export function createResponseConversationReplacement( + mongoose: typeof import('mongoose'), +): (owner: ResponseOwner, conversationId: string) => Promise { + let supported: boolean | null = null; + return async (owner, conversationId) => { + supported = await getTransactionSupport(mongoose, supported); + const records = createStoredResponseModel(mongoose); + const filter = { ...responseOwnerFilter(owner), conversationId }; + if (!supported) { + // Reservation fails closed on standalone deployments, so no admission can race this check. + if (await records.exists(filter)) return false; + await closeResponseScopes(mongoose, owner, [conversationId]); + return true; + } + const key = responseScopeKeys(owner, conversationId)[1]; + await initializeResponseScopes(mongoose, [key]); + const session = await mongoose.startSession(); + try { + let allowed = false; + await session.withTransaction(async () => { + const scopes = createResponseScopeModel(mongoose); + // Serialize with the reservation's write to this same anchor before checking rows. + await scopes.updateOne({ _id: key }, { $inc: { revision: 1 } }, { session }); + if (await records.exists(filter).session(session)) { + allowed = false; + return; + } + await scopes.updateOne({ _id: key }, { $set: { closed: true } }, { session }); + allowed = true; + }); + return allowed; + } finally { + await session.endSession(); + } + }; +} + +/** Authorized missing-root recovery, restricted to one globally unique owner/conversation. */ +export async function discoverResponseOwners( + mongoose: typeof import('mongoose'), + userId: string, + conversationId: string, +): Promise { + return runAsSystem(async () => { + const model = createStoredResponseModel(mongoose); + const filter = { userId, conversationId }; + const [tenants, tenantless] = await Promise.all([ + model.distinct('tenantId', filter), + model.exists({ ...filter, tenantId: { $in: [null, ''] } }), + ]); + const owners: ResponseOwner[] = tenants + .filter((tenantId): tenantId is string => typeof tenantId === 'string' && tenantId !== '') + .map((tenantId) => ({ userId, tenantId })); + if (tenantless) owners.push({ userId }); + return owners; + }); +} + +/** Retention intent is monotonic, like erasure: failed parent saves never reopen private data. */ +export function createResponseRetention( + mongoose: typeof import('mongoose'), +): (owner: ResponseOwner, conversationId: string, expiredAt: Date) => Promise { + let supported: boolean | null = null; + let indexes: Promise | undefined; + return async (owner, conversationId, expiredAt) => { + if (!(expiredAt instanceof Date) || !Number.isFinite(expiredAt.getTime())) + throw new TypeError('Invalid response expiration'); + const key = responseScopeKeys(owner, conversationId)[1]; + await initializeResponseScopes(mongoose, [key]); + const scopes = createResponseScopeModel(mongoose); + const records = createStoredResponseModel(mongoose); + supported = await getTransactionSupport(mongoose, supported); + if (supported) { + indexes ??= createIndexesWithRetry(records) + .then(() => undefined) + .catch((error: unknown) => { + indexes = undefined; + throw error; + }); + await indexes; + } + const apply = async (session?: import('mongoose').ClientSession) => { + const anchor = await scopes + .findOneAndUpdate( + { _id: key }, + { $min: { expiredAt }, $inc: { revision: 1 } }, + { new: true, ...(session ? { session } : {}) }, + ) + .lean(); + if (!anchor?.expiredAt) throw new Error('Response retention scope unavailable'); + await records.updateMany( + { + ...responseOwnerFilter(owner), + conversationId, + $or: [{ expiredAt: null }, { expiredAt: { $gt: anchor.expiredAt } }], + }, + { $set: { expiredAt: anchor.expiredAt } }, + session ? { session } : {}, + ); + }; + if (!supported) return apply(); + const session = await mongoose.startSession(); + try { + await session.withTransaction(() => apply(session)); + } finally { + await session.endSession(); + } + }; +} + +/** The tenant policy treats missing, null and empty tenant IDs as one partition. */ +export function responseOwnerFilter(owner: ResponseOwner): { + userId: string; + tenantId: string | { $in: Array }; +} { + return { userId: owner.userId, tenantId: owner.tenantId || { $in: [null, ''] } }; +} + +export async function eraseResponseRecords( + mongoose: typeof import('mongoose'), + owner: ResponseOwner, + conversationIds?: readonly string[], +): Promise { + await createStoredResponseModel(mongoose).deleteMany({ + ...responseOwnerFilter(owner), + ...(conversationIds ? { conversationId: { $in: conversationIds } } : {}), + }); +} + +function scopeKey(owner: ResponseOwner, conversationId?: string): string { + return createHash('sha256') + .update(JSON.stringify([owner.tenantId || null, owner.userId, conversationId ?? null])) + .digest('hex'); +} + +export function responseScopeKeys(owner: ResponseOwner, conversationId: string): string[] { + return [scopeKey(owner), scopeKey(owner, conversationId)]; +} + +/** Initializes only metadata; a late initializer can never reopen an erased scope. */ +export async function initializeResponseScopes( + mongoose: typeof import('mongoose'), + keys: string[], +): Promise { + const model = createResponseScopeModel(mongoose); + for (const key of keys) { + try { + await model.updateOne( + { _id: key }, + { $setOnInsert: { closed: false, revision: 0 } }, + { upsert: true }, + ); + } catch (error) { + if ( + !(error instanceof mongoose.mongo.MongoServerError) || + error.code !== 11000 || + !(await model.exists({ _id: key })) + ) + throw error; + } + } +} + +export async function isResponseScopeClosed( + mongoose: typeof import('mongoose'), + owner: ResponseOwner, + conversationId: string, +): Promise { + return ( + (await createResponseScopeModel(mongoose).exists({ + _id: { $in: responseScopeKeys(owner, conversationId) }, + closed: true, + })) != null + ); +} + +/** Irreversible deletion intent. Ordinary retries complete erasure without readmitting work. */ +export async function closeResponseScopes( + mongoose: typeof import('mongoose'), + owner: ResponseOwner, + conversationIds?: string[], +): Promise { + const model = createResponseScopeModel(mongoose); + const keys = conversationIds?.map((id) => scopeKey(owner, id)) ?? [scopeKey(owner)]; + // Bound each command independently of the number of conversations being removed. + for (let offset = 0; offset < keys.length; offset += 250) { + const operations = keys.slice(offset, offset + 250).map((key) => ({ + updateOne: { + filter: { _id: key }, + update: { $set: { closed: true }, $setOnInsert: { revision: 0 } }, + upsert: true, + }, + })); + // Keys already hash tenant + owner; these payload-free anchors have no tenantId column. + // eslint-disable-next-line no-restricted-syntax + const close = () => model.bulkWrite(operations, { ordered: false }); + try { + await close(); + } catch (error) { + if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 11000) throw error; + await close(); + } + } + await createStoredResponseModel(mongoose).updateMany( + { + ...responseOwnerFilter(owner), + ...(conversationIds ? { conversationId: { $in: conversationIds } } : {}), + status: { $in: ['queued', 'in_progress', 'requires_action'] }, + cancellationRequestedAt: { $exists: false }, + pendingTerminal: { $exists: false }, + }, + { $set: { cancellationRequestedAt: Date.now() } }, + ); +} diff --git a/packages/data-schemas/src/methods/test-helpers.spec.ts b/packages/data-schemas/src/methods/test-helpers.spec.ts new file mode 100644 index 0000000000..9380dc343f --- /dev/null +++ b/packages/data-schemas/src/methods/test-helpers.spec.ts @@ -0,0 +1,30 @@ +import { waitForPause } from './test-helpers'; + +it('reaches a pause without waiting for the blocked operation', async () => { + await expect( + waitForPause(Promise.resolve(), new Promise(() => undefined)), + ).resolves.toBeUndefined(); +}); + +it('rejects an operation that completes without reaching its pause', async () => { + await expect( + waitForPause(new Promise(() => undefined), Promise.resolve('finished')), + ).rejects.toThrow('Operation completed before reaching its pause'); +}); + +it('preserves an operation failure before its pause', async () => { + const failure = new Error('write failed'); + await expect( + waitForPause(new Promise(() => undefined), Promise.reject(failure)), + ).rejects.toBe(failure); +}); + +it('observes a late operation failure after its pause was reached', async () => { + let fail!: (reason: Error) => void; + const operation = new Promise((_resolve, reject) => { + fail = reject; + }); + await expect(waitForPause(Promise.resolve(), operation)).resolves.toBeUndefined(); + fail(new Error('late write failure')); + await new Promise((resolve) => setImmediate(resolve)); +}); diff --git a/packages/data-schemas/src/methods/test-helpers.ts b/packages/data-schemas/src/methods/test-helpers.ts index e7c52051a3..e430655c23 100644 --- a/packages/data-schemas/src/methods/test-helpers.ts +++ b/packages/data-schemas/src/methods/test-helpers.ts @@ -1,5 +1,15 @@ import { gptPointReleaseFamily } from 'librechat-data-provider'; +/** Fail promptly if an operation settles without reaching the expected pause. */ +export function waitForPause(pause: Promise, operation: Promise): Promise { + return Promise.race([ + pause, + operation.then(() => { + throw new Error('Operation completed before reaching its pause'); + }), + ]); +} + /** * Inlined utility functions previously imported from @librechat/api. * These are used only by test files in data-schemas. diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index d86d27e728..cab7c8080b 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -8,6 +8,9 @@ import { import type * as t from '~/types'; import { createToolApprovalGrantModel } from '~/models/toolApprovalGrant'; import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user'; +import { createStoredResponseModel } from '~/models/response'; +import { runAsSystem } from '~/config/tenantContext'; +import { createResponseStorage } from './response'; import balanceSchema from '~/schema/balance'; import userSchema from '~/schema/user'; @@ -2362,3 +2365,85 @@ describe('personal grant cleanup before account deletion', () => { } }); }); + +describe('private response cleanup before account deletion', () => { + async function createResponses() { + const user = await User.create({ email: 'delete-responses@example.com', provider: 'local' }); + const userId = user._id.toString(); + const model = createStoredResponseModel(mongoose); + await runAsSystem(() => + model.create( + [{ userId }, { userId, tenantId: 'another-tenant' }, { userId: 'another-owner' }].map( + (scope, index) => ({ + ...scope, + responseId: `response-${index}`, + requestKey: `request-${index}`, + fingerprint: 'request', + agentId: 'agent', + requestJSON: '{"input":"private request"}', + responseJSON: '{"output":"private reply"}', + conversationId: 'conversation', + parentMessageId: 'root', + status: 'completed', + }), + ), + ), + ); + return { userId, model }; + } + + it('removes private payloads before committing deletion and preserves other owners and tenants', async () => { + const { userId, model } = await createResponses(); + const cleanup = jest.spyOn(model, 'deleteMany'); + const deletion = jest.spyOn(User, 'deleteOne'); + await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 }); + expect(cleanup).toHaveBeenCalledWith({ userId, tenantId: { $in: [null, ''] } }); + expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(deletion.mock.invocationCallOrder[0]); + await runAsSystem(async () => { + expect(await model.countDocuments({ userId, tenantId: null })).toBe(0); + expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(1); + expect(await model.countDocuments({ userId: 'another-owner' })).toBe(1); + }); + }); + + it('erases every tenantless spelling when the legacy account stores an empty tenant', async () => { + const { userId, model } = await createResponses(); + await User.updateOne({ _id: userId }, { $set: { tenantId: '' } }); + const original = (await model.findOne({ userId, responseId: 'response-0' }).lean())!; + const { _id: _originalId, ...fields } = original; + await runAsSystem(async () => { + await model.create([ + { ...fields, tenantId: '', responseId: 'empty-tenant', requestKey: 'empty-tenant' }, + { ...fields, tenantId: null, responseId: 'null-tenant', requestKey: 'null-tenant' }, + ]); + await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 }); + expect(await model.countDocuments({ userId, tenantId: { $in: [null, ''] } })).toBe(0); + expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(1); + }); + }); + + it('preserves the account on cleanup failure and allows a retry', async () => { + const { userId, model } = await createResponses(); + jest + .spyOn(model.collection, 'deleteMany') + .mockRejectedValueOnce(new Error('response-store down')); + const deletion = jest.spyOn(User, 'deleteOne'); + await expect(methods.deleteUserById(userId)).rejects.toThrow('response-store down'); + expect(deletion).not.toHaveBeenCalled(); + expect(await User.exists({ _id: userId })).not.toBeNull(); + expect(await createResponseStorage(mongoose).read({ userId }, 'response-0')).not.toBeNull(); + await methods.updateUser(userId, { name: 'Still usable' }); + expect((await User.findById(userId).lean())?.name).toBe('Still usable'); + await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 }); + }); + + it('uses the account tenant when deletion runs under system authority', async () => { + const { userId, model } = await createResponses(); + await User.updateOne({ _id: userId }, { $set: { tenantId: 'another-tenant' } }); + await runAsSystem(async () => { + await expect(methods.deleteUserById(userId)).resolves.toMatchObject({ deletedCount: 1 }); + expect(await model.countDocuments({ userId, tenantId: 'another-tenant' })).toBe(0); + expect(await model.countDocuments({ userId, tenantId: null })).toBe(1); + }); + }); +}); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 8d5d444c37..78dddcc202 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -16,6 +16,7 @@ import type { } from '~/types'; import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types'; import type { CacheStore } from '~/types'; +import { closeResponseScopes, eraseResponseRecords } from './responseScope'; import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search'; import { evictAuthUserDocsForUsers } from '~/utils/eviction'; import { signPayload } from '~/crypto'; @@ -641,6 +642,11 @@ export function createUserMethods( async function deleteUserById(userId: string): Promise { try { const User = mongoose.models.User; + const owner = await User.findById(userId).select('tenantId').lean>(); + if (owner) { + await closeResponseScopes(mongoose, { userId, tenantId: owner.tenantId }); + await eraseResponseRecords(mongoose, { userId, tenantId: owner.tenantId }); + } await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); const result = await User.deleteOne({ _id: userId }); if (result.deletedCount === 0) { diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts index 8081a9310a..badff5d2cc 100644 --- a/packages/data-schemas/src/models/index.ts +++ b/packages/data-schemas/src/models/index.ts @@ -1,4 +1,5 @@ import { createAgentQueuedTurnModel, createAgentQueuedTurnSequenceModel } from './queuedTurn'; + import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence'; import { createScheduleModel, createScheduleRunModel } from './schedule'; import { getTenantIndexMigrationHint } from '~/migrations/tenantIndexes'; @@ -10,12 +11,15 @@ import { createAgentTriggerDeliveryModel } from './triggerDelivery'; import { createToolApprovalGrantModel } from './toolApprovalGrant'; import { createSkillUploadIntentModel } from './skillUploadIntent'; import { createSubagentContentModel } from './subagentContent'; + import { createSkillSyncStatusModel } from './skillSyncStatus'; import { createConversationTagModel } from './conversationTag'; import { createCodeEnvironmentModel } from './codeEnvironment'; import { createSkillDeletionModel } from './skillDeletion'; import { createAgentCategoryModel } from './agentCategory'; +import { createResponseScopeModel } from './responseScope'; import { createResourceLockModel } from './resourceLock'; +import { createStoredResponseModel } from './response'; import { createChatProjectModel } from './chatProject'; import { createAgentApiKeyModel } from './agentApiKey'; import { createTransactionModel } from './transaction'; @@ -57,6 +61,8 @@ import logger from '~/config/winston'; */ export function createModels(mongoose: typeof import('mongoose')): { User: ReturnType; + StoredResponse: ReturnType; + ResponseScope: ReturnType; ToolApprovalGrant: ReturnType; Token: ReturnType; Session: ReturnType; @@ -112,6 +118,8 @@ export function createModels(mongoose: typeof import('mongoose')): { } { const models = { User: createUserModel(mongoose), + StoredResponse: createStoredResponseModel(mongoose), + ResponseScope: createResponseScopeModel(mongoose), ToolApprovalGrant: createToolApprovalGrantModel(mongoose), Token: createTokenModel(mongoose), Session: createSessionModel(mongoose), diff --git a/packages/data-schemas/src/models/response.ts b/packages/data-schemas/src/models/response.ts new file mode 100644 index 0000000000..93e98d38b7 --- /dev/null +++ b/packages/data-schemas/src/models/response.ts @@ -0,0 +1,195 @@ +import { Schema } from 'mongoose'; +import type { Model } from 'mongoose'; +import { applyTenantIsolation } from './plugins/tenantIsolation'; + +export interface ResponseOwner { + userId: string; + tenantId?: string; +} + +export type StoredResponseStatus = + | 'queued' + | 'in_progress' + | 'requires_action' + | 'completed' + | 'failed' + | 'incomplete' + | 'cancelled'; + +export interface ResponseReviewCheckpoint { + actionId: string; + providerExecutionId: string; + expiresAt: number; + /** Private seed for the resumed output; never part of the public response view. */ + partialResponseJSON: string; + /** Visible assistant token counter, separate from billed multi-model usage. */ + visibleOutputTokens?: number; + /** Private reviewed graph/tool/checkpoint bindings retained after the job consumes its action. */ + pendingActionJSON?: string; + /** Exact checkpointer settings captured by the paused run, independent of later config changes. */ + checkpointConfigJSON?: string; + checkpointId?: string; + checkpointNs?: string; +} + +export interface ResponseResumeIntent { + intentId: string; + actionId: string; + previousProviderExecutionId: string; + providerExecutionId: string; + fingerprint: string; + /** Validated, owner-submitted SDK resolution; not a credential or executable request. */ + resolutionJSON: string; + /** Original validated browser decision, including edited arguments and remembered-approval scope. */ + decisionJSON?: string; + state: 'reserved' | 'admitted'; + reservedAt: number; + admittedAt?: number; +} + +/** Server-written receipt after provider and trailing writes have definitively settled. */ +export interface ResponseTerminalReceipt { + generationCreatedAt: number; + providerExecutionId: string; + ownerId?: string; + status: 'completed' | 'failed' | 'incomplete' | 'cancelled'; + responseJSON: string; + checkpointNamespace?: string; + /** Server-pinned saver settings, including initial failures before a review exists. */ + checkpointConfigJSON?: string; +} + +/** Durable identity and dispatch/result journal for a background Responses request. */ +export interface StoredResponseRecord { + expiredAt?: Date; + isTemporary?: boolean; + tenantId?: string; + userId: string; + responseId: string; + requestKey: string; + fingerprint: string; + agentId: string; + sourceKeyId?: string; + requestJSON: string; + conversationId: string; + parentMessageId: string; + status: StoredResponseStatus; + revision: number; + deliveryKey?: string; + generationCreatedAt?: number; + responseJSON?: string; + cancellationRequestedAt?: number; + review?: ResponseReviewCheckpoint; + resume?: ResponseResumeIntent; + pendingTerminal?: ResponseTerminalReceipt; + createdAt: Date; + updatedAt: Date; +} + +export function createStoredResponseModel( + mongoose: typeof import('mongoose'), +): Model { + const schema = new Schema( + { + tenantId: String, + expiredAt: Date, + isTemporary: Boolean, + userId: { type: String, required: true }, + responseId: { type: String, required: true }, + requestKey: { type: String, required: true }, + fingerprint: { type: String, required: true }, + agentId: { type: String, required: true }, + sourceKeyId: String, + requestJSON: { type: String, required: true }, + conversationId: { type: String, required: true }, + parentMessageId: { type: String, required: true }, + status: { + type: String, + required: true, + enum: [ + 'queued', + 'in_progress', + 'requires_action', + 'completed', + 'failed', + 'incomplete', + 'cancelled', + ], + }, + revision: { type: Number, required: true, default: 0 }, + deliveryKey: String, + generationCreatedAt: Number, + responseJSON: String, + cancellationRequestedAt: Number, + pendingTerminal: { + type: new Schema( + { + generationCreatedAt: { type: Number, required: true, min: 1 }, + providerExecutionId: { type: String, required: true, maxlength: 128 }, + ownerId: { type: String, maxlength: 128 }, + status: { + type: String, + required: true, + enum: ['completed', 'failed', 'incomplete', 'cancelled'], + }, + responseJSON: { type: String, required: true }, + checkpointNamespace: String, + checkpointConfigJSON: String, + }, + { _id: false }, + ), + required: false, + }, + review: { + type: new Schema( + { + actionId: { type: String, required: true, maxlength: 256 }, + providerExecutionId: { type: String, required: true, maxlength: 128 }, + expiresAt: { type: Number, required: true, min: 1 }, + partialResponseJSON: { type: String, required: true }, + visibleOutputTokens: { + type: Number, + min: 0, + max: Number.MAX_SAFE_INTEGER, + validate: Number.isSafeInteger, + }, + pendingActionJSON: String, + checkpointConfigJSON: String, + checkpointId: String, + checkpointNs: String, + }, + { _id: false }, + ), + required: false, + }, + resume: { + type: new Schema( + { + intentId: { type: String, required: true, maxlength: 256 }, + actionId: { type: String, required: true, maxlength: 256 }, + previousProviderExecutionId: { type: String, required: true, maxlength: 128 }, + providerExecutionId: { type: String, required: true, maxlength: 128 }, + fingerprint: { type: String, required: true }, + resolutionJSON: { type: String, required: true }, + decisionJSON: String, + state: { type: String, required: true, enum: ['reserved', 'admitted'] }, + reservedAt: { type: Number, required: true, min: 1 }, + admittedAt: { type: Number, min: 1 }, + }, + { _id: false }, + ), + required: false, + }, + }, + { timestamps: true, minimize: false }, + ); + schema.index({ tenantId: 1, userId: 1, responseId: 1 }, { unique: true }); + schema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 }); + schema.index({ tenantId: 1, userId: 1, requestKey: 1 }, { unique: true }); + schema.index({ tenantId: 1, userId: 1, conversationId: 1, createdAt: 1 }); + schema.index({ userId: 1, conversationId: 1, tenantId: 1 }); + applyTenantIsolation(schema); + return ( + mongoose.models.StoredResponse ?? mongoose.model('StoredResponse', schema) + ); +} diff --git a/packages/data-schemas/src/models/responseScope.ts b/packages/data-schemas/src/models/responseScope.ts new file mode 100644 index 0000000000..a705b8d227 --- /dev/null +++ b/packages/data-schemas/src/models/responseScope.ts @@ -0,0 +1,26 @@ +import { Schema } from 'mongoose'; +import type { Model } from 'mongoose'; + +interface ResponseScope { + _id: string; + closed: boolean; + revision: number; + expiredAt?: Date; +} + +/** Payload-free admission anchors; closed scopes remain tombstones after erasure. */ +export function createResponseScopeModel( + mongoose: typeof import('mongoose'), +): Model { + if (mongoose.models.ResponseScope) return mongoose.model('ResponseScope'); + const schema = new Schema( + { + _id: { type: String, required: true }, + closed: { type: Boolean, required: true, default: false }, + revision: { type: Number, required: true, default: 0 }, + expiredAt: Date, + }, + { versionKey: false }, + ); + return mongoose.model('ResponseScope', schema); +} diff --git a/packages/data-schemas/src/utils/transactions.ts b/packages/data-schemas/src/utils/transactions.ts index bc1ed55835..aeb2b6295d 100644 --- a/packages/data-schemas/src/utils/transactions.ts +++ b/packages/data-schemas/src/utils/transactions.ts @@ -1,105 +1,76 @@ import logger from '~/config/winston'; export const CANCEL_RATE = 1.15; - const TRANSACTION_PROBE_COLLECTION = '__transaction_test__'; +function unsupportedTransaction(error: unknown, mongoose: typeof import('mongoose')): boolean { + return ( + error instanceof mongoose.mongo.MongoServerError && + error.code === 20 && + /Transaction numbers are only allowed on a replica set member or mongos/i.test(error.message) + ); +} + async function probeTransaction(mongoose: typeof import('mongoose')): Promise { + const session = await mongoose.startSession(); try { - const session = await mongoose.startSession(); - try { - session.startTransaction(); - - await mongoose.connection.db - ?.collection(TRANSACTION_PROBE_COLLECTION) - .findOne({}, { session }); - - await session.commitTransaction(); - logger.debug('MongoDB transactions are supported'); - return true; - } catch (transactionError: unknown) { - try { - await session.abortTransaction(); - } catch (transactionError) { - /** best-effort abort */ - logger.error(`[supportsTransactions] Error aborting transaction:`, transactionError); - } - logger.debug( - 'MongoDB transactions not supported (transaction error):', - (transactionError as Error)?.message || 'Unknown error', - ); - return false; - } finally { - await session.endSession(); - } + session.startTransaction(); + const db = mongoose.connection.db; + if (!db) throw new Error('Transaction capability probe requires a connected database'); + await db.collection(TRANSACTION_PROBE_COLLECTION).findOne({}, { session }); + await session.commitTransaction(); + return true; } catch (error) { - logger.debug( - 'MongoDB transactions not supported (session error):', - (error as Error)?.message || 'Unknown error', - ); - return false; + try { + await session.abortTransaction(); + } catch { + logger.debug('[supportsTransactions] Probe transaction abort failed'); + } + if (unsupportedTransaction(error, mongoose)) return false; + throw error; + } finally { + await session.endSession(); } } -/** - * Checks if the connected MongoDB deployment supports transactions - * This requires a MongoDB replica set configuration - * - * Amazon DocumentDB rejects a transaction that touches a collection which does - * not exist ("Feature not supported: non-existent collection in transaction"), - * so a failed first probe materializes the canary and probes once more — - * otherwise engines that fully support transactions would report unsupported - * and silently drop every caller to the non-transactional path. MongoDB allows - * transactional reads of missing collections, so deployments that pass the - * first probe never pay the create and never gain the canary collection. A - * failed create is logged rather than swallowed: on a hardened role it is the - * only visible explanation for a wrong "unsupported" verdict. - * - * @returns True if transactions are supported, false otherwise - */ +/** Only confirmed standalone incompatibility is false; operational failures stay retryable. */ export const supportsTransactions = async ( mongoose: typeof import('mongoose'), ): Promise => { - if (await probeTransaction(mongoose)) { - return true; - } try { - await mongoose.connection.db?.createCollection(TRANSACTION_PROBE_COLLECTION); + return await probeTransaction(mongoose); } catch (error) { - const message = (error as Error)?.message ?? ''; - if (!/already exists|NamespaceExists/i.test(message)) { - logger.warn( - `[supportsTransactions] Could not create the "${TRANSACTION_PROBE_COLLECTION}" probe collection; ` + - 'if this deployment is Amazon DocumentDB, the transaction-support verdict below may be a false negative:', - error, - ); + // DocumentDB requires the probe collection to exist before a transaction can read it. + if ( + !(error instanceof mongoose.mongo.MongoServerError) || + !/non-existent collection in transaction/i.test(error.message) + ) + throw error; + const db = mongoose.connection.db; + if (!db) throw error; + try { + await db.createCollection(TRANSACTION_PROBE_COLLECTION); + } catch (creationError) { + if (!(creationError instanceof mongoose.mongo.MongoServerError) || creationError.code !== 48) + throw creationError; } + return probeTransaction(mongoose); } - return probeTransaction(mongoose); }; -let inflightProbe: Promise | null = null; +const inflightProbes = new WeakMap>(); -/** - * Gets whether the current MongoDB deployment supports transactions - * Caches the result for performance - * - * Concurrent first callers share one in-flight probe instead of each paying - * the session/transaction round trips before the caller-side cache fills. - * - * @returns True if transactions are supported, false otherwise - */ +/** Share first probes per connection. Callers cache only a successful capability verdict. */ export const getTransactionSupport = async ( mongoose: typeof import('mongoose'), transactionSupportCache: boolean | null, ): Promise => { - if (transactionSupportCache !== null) { - return transactionSupportCache; + if (transactionSupportCache !== null) return transactionSupportCache; + const connection = mongoose.connection; + let probe = inflightProbes.get(connection); + if (!probe) { + probe = supportsTransactions(mongoose).finally(() => inflightProbes.delete(connection)); + inflightProbes.set(connection, probe); } - if (inflightProbe == null) { - inflightProbe = supportsTransactions(mongoose).finally(() => { - inflightProbe = null; - }); - } - return await inflightProbe; + return probe; };