diff --git a/api/models/index.js b/api/models/index.js index 4ab4bfc6aa..49a550e786 100644 --- a/api/models/index.js +++ b/api/models/index.js @@ -5,16 +5,19 @@ const { findMatchingPattern, isDeploymentSkillId, createMessageBudgetReader, + createChannelStorageConfigReader, } = require('@librechat/api'); const getLogStores = require('~/cache/getLogStores'); const messageBudget = createMessageBudgetReader(); +const channelStorageConfig = createChannelStorageConfigReader(); const methods = createMethods(mongoose, { matchModelName, findMatchingPattern, isExternalSkillId: isDeploymentSkillId, getCache: getLogStores, + getChannelStorageConfig: channelStorageConfig.getConfig, getMCPAppMessageBudget: messageBudget.getBudget, }); @@ -28,5 +31,6 @@ const seedDatabase = async () => { module.exports = { ...methods, initializeMessageBudget: messageBudget.initialize, + initializeChannelStorageConfig: channelStorageConfig.initialize, seedDatabase, }; diff --git a/api/models/index.spec.js b/api/models/index.spec.js index a5bc4c9b41..d4f0aa369b 100644 --- a/api/models/index.spec.js +++ b/api/models/index.spec.js @@ -3,15 +3,21 @@ jest.mock('@librechat/api', () => ({ matchModelName: jest.fn(), findMatchingPattern: jest.fn(), isDeploymentSkillId: jest.fn(), + createChannelStorageConfigReader: jest.fn(() => ({ + getConfig: jest.fn(), + initialize: jest.fn(), + })), createMessageBudgetReader: jest.fn(() => ({ getBudget: jest.fn(), initialize: jest.fn() })), })); jest.mock('~/cache/getLogStores', () => jest.fn()); const { createMethods } = require('@librechat/data-schemas'); -const { createMessageBudgetReader } = require('@librechat/api'); -const { initializeMessageBudget } = require('./index'); +const { createMessageBudgetReader, createChannelStorageConfigReader } = require('@librechat/api'); +const { initializeMessageBudget, initializeChannelStorageConfig } = require('./index'); const reader = createMessageBudgetReader.mock.results[0].value; -const { getMCPAppMessageBudget } = createMethods.mock.calls[0][1]; +const channelStorageConfigReader = createChannelStorageConfigReader.mock.results[0].value; +const { getMCPAppMessageBudget, getChannelStorageConfig: channelStorageConfigDependency } = + createMethods.mock.calls[0][1]; describe('message App budget dependency wiring', () => { it('passes the same reader to storage and initializer to the composition root', () => { @@ -20,3 +26,11 @@ describe('message App budget dependency wiring', () => { expect(reader.getBudget).not.toHaveBeenCalled(); }); }); + +describe('channel storage configuration dependency wiring', () => { + it('injects the deployment reader without reading config during module initialization', () => { + expect(channelStorageConfigDependency).toBe(channelStorageConfigReader.getConfig); + expect(initializeChannelStorageConfig).toBe(channelStorageConfigReader.initialize); + expect(channelStorageConfigReader.getConfig).not.toHaveBeenCalled(); + }); +}); diff --git a/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js b/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js index f7117de240..04b7dde0d0 100644 --- a/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js +++ b/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js @@ -35,6 +35,7 @@ jest.mock('@librechat/api', () => ({ jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), saveMessage: jest.fn(async (req, message) => message), getConvo: jest.fn(async () => null), getMessages: jest.fn(async () => []), diff --git a/api/server/controllers/agents/__tests__/client.subagentUsage.spec.js b/api/server/controllers/agents/__tests__/client.subagentUsage.spec.js index 3fd3152fc9..c9b24becbf 100644 --- a/api/server/controllers/agents/__tests__/client.subagentUsage.spec.js +++ b/api/server/controllers/agents/__tests__/client.subagentUsage.spec.js @@ -3,6 +3,7 @@ const mockGetCacheMultiplier = jest.fn(() => 1); jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), getMultiplier: (...args) => mockGetMultiplier(...args), getCacheMultiplier: (...args) => mockGetCacheMultiplier(...args), })); diff --git a/api/server/controllers/agents/__tests__/client.transactions.spec.js b/api/server/controllers/agents/__tests__/client.transactions.spec.js index 4ad3d45172..d9cb696419 100644 --- a/api/server/controllers/agents/__tests__/client.transactions.spec.js +++ b/api/server/controllers/agents/__tests__/client.transactions.spec.js @@ -2,6 +2,7 @@ const mockSpendTokens = jest.fn().mockResolvedValue(undefined); jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), spendTokens: mockSpendTokens, })); diff --git a/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js b/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js index 365438dcab..3b8f160e9e 100644 --- a/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js +++ b/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js @@ -33,6 +33,7 @@ jest.mock('@librechat/api', () => ({ jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), saveMessage: jest.fn(async (req, message) => message), getConvo: jest.fn(async () => null), getMessages: jest.fn(async () => []), diff --git a/api/server/controllers/agents/recordCollectedUsage.spec.js b/api/server/controllers/agents/recordCollectedUsage.spec.js index 4e86cef751..18380da693 100644 --- a/api/server/controllers/agents/recordCollectedUsage.spec.js +++ b/api/server/controllers/agents/recordCollectedUsage.spec.js @@ -20,6 +20,7 @@ const mockRecordCollectedUsage = jest jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), spendTokens: (...args) => mockSpendTokens(...args), spendStructuredTokens: (...args) => mockSpendStructuredTokens(...args), getMultiplier: mockGetMultiplier, diff --git a/api/server/routes/agents/__tests__/abort.spec.js b/api/server/routes/agents/__tests__/abort.spec.js index d083ece6bc..6655b8e947 100644 --- a/api/server/routes/agents/__tests__/abort.spec.js +++ b/api/server/routes/agents/__tests__/abort.spec.js @@ -51,6 +51,7 @@ jest.mock('@librechat/api', () => ({ jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), saveMessage: (...args) => mockSaveMessage(...args), getPersistedPrivateTextId: async (...args) => (await mockHasPersistedPrivateText(...args)) ? 'protected-row-id' : null, diff --git a/api/server/routes/agents/__tests__/streamTenant.spec.js b/api/server/routes/agents/__tests__/streamTenant.spec.js index e151c9fd54..e96b74711a 100644 --- a/api/server/routes/agents/__tests__/streamTenant.spec.js +++ b/api/server/routes/agents/__tests__/streamTenant.spec.js @@ -39,6 +39,7 @@ jest.mock('@librechat/api', () => ({ jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), saveMessage: (...args) => mockSaveMessage(...args), })); @@ -86,6 +87,28 @@ function mockSubscribeSuccess() { }); } +/** A real HTTP socket must reach subscribe before fake time can exercise its keepalive. */ +function openSilentStream() { + let resolveAttached; + const subscribed = new Promise((resolve) => { + resolveAttached = resolve; + }); + mockGenerationJobManager.subscribe.mockImplementation(async (_streamId, _writeEvent, onDone) => { + resolveAttached(onDone); + return { unsubscribe: jest.fn() }; + }); + const pending = request(app) + .get('/agents/chat/stream/stream-123') + .then((res) => res); + const attached = Promise.race([ + subscribed, + pending.then((res) => { + throw new Error(`Stream ended before subscribing: HTTP ${res.status}`); + }), + ]); + return { pending, attached }; +} + describe('SSE stream tenant isolation', () => { beforeEach(() => { jest.clearAllMocks(); @@ -411,19 +434,8 @@ describe('SSE stream tenant isolation', () => { status: 'running', createdAt: 1000, }); - let finish; - mockGenerationJobManager.subscribe.mockImplementation( - async (_streamId, _writeEvent, onDone) => { - finish = onDone; - return { unsubscribe: jest.fn() }; - }, - ); - - const pending = request(app) - .get('/agents/chat/stream/stream-123') - .then((res) => res); - await jest.advanceTimersByTimeAsync(0); - await new Promise((resolve) => setImmediate(resolve)); + const { pending, attached } = openSilentStream(); + const finish = await attached; await jest.advanceTimersByTimeAsync(60_000); finish({ final: true }); const res = await pending; @@ -470,19 +482,8 @@ describe('SSE stream tenant isolation', () => { status: 'running', createdAt: 1000, }); - let finish; - mockGenerationJobManager.subscribe.mockImplementation( - async (_streamId, _writeEvent, onDone) => { - finish = onDone; - return { unsubscribe: jest.fn() }; - }, - ); - - const pending = request(app) - .get('/agents/chat/stream/stream-123') - .then((res) => res); - await jest.advanceTimersByTimeAsync(0); - await new Promise((resolve) => setImmediate(resolve)); + const { pending, attached } = openSilentStream(); + const finish = await attached; await jest.advanceTimersByTimeAsync(60_000); finish({ final: true }); const res = await pending; @@ -502,19 +503,8 @@ describe('SSE stream tenant isolation', () => { status: 'running', createdAt: 1000, }); - let finish; - mockGenerationJobManager.subscribe.mockImplementation( - async (_streamId, _writeEvent, onDone) => { - finish = onDone; - return { unsubscribe: jest.fn() }; - }, - ); - - const pending = request(app) - .get('/agents/chat/stream/stream-123') - .then((res) => res); - await jest.advanceTimersByTimeAsync(0); - await new Promise((resolve) => setImmediate(resolve)); + const { pending, attached } = openSilentStream(); + const finish = await attached; await jest.advanceTimersByTimeAsync(60_000); expect(mockGenerationJobManager.subscribe).toHaveBeenCalled(); diff --git a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js index 3e24e018ac..c0713198c8 100644 --- a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js +++ b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js @@ -17,6 +17,7 @@ jest.mock('~/server/services/start/tools', () => ({ jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), getApplicableConfigs: jest.fn().mockResolvedValue([]), getUserPrincipals: jest.fn().mockResolvedValue([]), })); diff --git a/api/server/services/Config/__tests__/invalidateConfigCaches.spec.js b/api/server/services/Config/__tests__/invalidateConfigCaches.spec.js index dee4cbd2af..7c34dcdf44 100644 --- a/api/server/services/Config/__tests__/invalidateConfigCaches.spec.js +++ b/api/server/services/Config/__tests__/invalidateConfigCaches.spec.js @@ -22,6 +22,7 @@ jest.mock('@librechat/data-schemas', () => { jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), getApplicableConfigs: jest.fn().mockResolvedValue([]), getUserPrincipals: jest.fn().mockResolvedValue([]), })); @@ -61,14 +62,16 @@ jest.mock('@librechat/api', () => ({ // ── Tests ────────────────────────────────────────────────────────────── const { getAppConfig, invalidateConfigCaches } = require('../app'); -const { initializeMessageBudget } = require('~/models'); +const { initializeMessageBudget, initializeChannelStorageConfig } = require('~/models'); // Capture module-composition calls before clearMocks resets the spy history. const budgetInitializations = [...initializeMessageBudget.mock.calls]; +const channelInitializations = [...initializeChannelStorageConfig.mock.calls]; const invalidationOptions = mockCreateConfigCacheInvalidation.mock.calls[0][0]; const pubSubOptions = mockCreateRedisPubSub.mock.calls[0][0]; test('Config supplies its actual reader exactly once during composition', () => { expect(budgetInitializations).toEqual([[getAppConfig]]); + expect(channelInitializations).toEqual([[getAppConfig]]); }); describe('invalidateConfigCaches', () => { diff --git a/api/server/services/Config/app.js b/api/server/services/Config/app.js index 665d4f1d95..4a56f6d61c 100644 --- a/api/server/services/Config/app.js +++ b/api/server/services/Config/app.js @@ -88,6 +88,7 @@ const { // Config owns the reader; models never import this module to obtain it. db.initializeMessageBudget(getAppConfig); +db.initializeChannelStorageConfig(getAppConfig); /** * Admin config writes clear the per-principal override caches, the MCP config-source servers diff --git a/api/server/services/Config/catalog.spec.js b/api/server/services/Config/catalog.spec.js index cbc7bd9334..84999229d0 100644 --- a/api/server/services/Config/catalog.spec.js +++ b/api/server/services/Config/catalog.spec.js @@ -3,7 +3,11 @@ jest.mock('@librechat/api', () => ({ fetchModels: jest.fn(), })); jest.mock('./app'); -jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn() })); +jest.mock('~/models', () => ({ + getUserKeyValues: jest.fn(), + initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), +})); jest.mock('@librechat/data-schemas', () => ({ ...jest.requireActual('@librechat/data-schemas'), logger: { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), info: jest.fn() }, diff --git a/api/server/services/Config/loadConfigModels.spec.js b/api/server/services/Config/loadConfigModels.spec.js index d391fc85a7..8b5de41da3 100644 --- a/api/server/services/Config/loadConfigModels.spec.js +++ b/api/server/services/Config/loadConfigModels.spec.js @@ -14,6 +14,7 @@ jest.mock('@librechat/data-schemas', () => ({ jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), })); const exampleConfig = { diff --git a/api/server/utils/import/importBatchBuilder.spec.js b/api/server/utils/import/importBatchBuilder.spec.js index 44ec73678a..120e5ab127 100644 --- a/api/server/utils/import/importBatchBuilder.spec.js +++ b/api/server/utils/import/importBatchBuilder.spec.js @@ -28,6 +28,7 @@ const { ImportBatchBuilder } = require('./importBatchBuilder'); jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), bulkIncrementTagCounts: jest.fn(), bulkSaveConvos: jest.fn(), bulkSaveMessages: jest.fn(), diff --git a/api/server/utils/import/importConversations.spec.js b/api/server/utils/import/importConversations.spec.js index 71b3702344..2f6a2d73ae 100644 --- a/api/server/utils/import/importConversations.spec.js +++ b/api/server/utils/import/importConversations.spec.js @@ -8,6 +8,7 @@ const importConversations = require('./importConversations'); jest.mock('~/models', () => ({ initializeMessageBudget: jest.fn(), + initializeChannelStorageConfig: jest.fn(), bulkIncrementTagCounts: jest.fn(), bulkSaveConvos: jest.fn(), bulkSaveMessages: jest.fn(), diff --git a/config/__tests__/migrate-shared-link-permissions.spec.js b/config/__tests__/migrate-shared-link-permissions.spec.js index 5d42dbebe8..a98d0bd6ef 100644 --- a/config/__tests__/migrate-shared-link-permissions.spec.js +++ b/config/__tests__/migrate-shared-link-permissions.spec.js @@ -1,6 +1,10 @@ jest.mock('../connect', () => jest.fn().mockResolvedValue(true)); jest.mock('@librechat/api', () => ({ ensureRequiredCollectionsExist: jest.fn().mockResolvedValue(undefined), + createChannelStorageConfigReader: jest.fn(() => ({ + initialize: jest.fn(), + getConfig: jest.fn(), + })), createMessageBudgetReader: jest.fn(() => ({ initialize: jest.fn(), getBudget: jest.fn() })), matchModelName: jest.fn(), findMatchingPattern: jest.fn(), diff --git a/packages/api/src/channels/config.spec.ts b/packages/api/src/channels/config.spec.ts new file mode 100644 index 0000000000..c15583e681 --- /dev/null +++ b/packages/api/src/channels/config.spec.ts @@ -0,0 +1,35 @@ +import type { AppConfig } from '@librechat/data-schemas'; +import { createChannelStorageConfigReader } from './config'; + +it('fails closed until the configuration service installs its reader', async () => { + await expect(createChannelStorageConfigReader().getConfig()).rejects.toThrow( + 'not been initialized', + ); +}); + +it('reads live base deployment bounds at each operation boundary', async () => { + const config = { endpoints: { agents: { channels: { erasureBatchSize: 2 } } } }; + const read = jest.fn(async () => config as AppConfig); + const effective = createChannelStorageConfigReader(); + effective.initialize(read); + expect(await effective.getConfig()).toMatchObject({ erasureBatchSize: 2, claimBatchSize: 25 }); + config.endpoints.agents.channels.erasureBatchSize = 3; + expect((await effective.getConfig()).erasureBatchSize).toBe(3); + expect(read).toHaveBeenCalledTimes(2); + expect(read).toHaveBeenLastCalledWith({ baseOnly: true }); + effective.initialize(async () => ({}) as AppConfig); + expect((await effective.getConfig()).erasureBatchSize).toBe(250); +}); + +it('rejects invalid bounds and propagates config outages', async () => { + const effective = createChannelStorageConfigReader(); + effective.initialize( + async () => ({ endpoints: { agents: { channels: { erasureBatchSize: 0 } } } }) as AppConfig, + ); + await expect(effective.getConfig()).rejects.toThrow(); + const failure = new Error('config unavailable'); + effective.initialize(async () => { + throw failure; + }); + await expect(effective.getConfig()).rejects.toBe(failure); +}); diff --git a/packages/api/src/channels/config.ts b/packages/api/src/channels/config.ts new file mode 100644 index 0000000000..157eb56729 --- /dev/null +++ b/packages/api/src/channels/config.ts @@ -0,0 +1,22 @@ +import { channelStorageConfigSchema } from 'librechat-data-provider'; +import type { ChannelStorageConfig } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { GetAppConfigOptions } from '~/app/service'; + +/** Config installs its reader during composition, without a models-to-config import cycle. */ +export function createChannelStorageConfigReader(): { + initialize: (reader: (options?: GetAppConfigOptions) => Promise) => void; + getConfig: () => Promise; +} { + let readConfig: ((options?: GetAppConfigOptions) => Promise) | undefined; + return { + initialize(reader) { + readConfig = reader; + }, + async getConfig() { + if (!readConfig) throw new Error('Channel storage config reader has not been initialized'); + const config = await readConfig({ baseOnly: true }); + return channelStorageConfigSchema.parse(config.endpoints?.agents?.channels ?? {}); + }, + }; +} diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 73e43cf2f0..30050e3f9e 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -155,3 +155,5 @@ export type * from './types'; export * from './tools/approval'; export * from './mcp/approval'; + +export { createChannelStorageConfigReader } from './channels/config'; diff --git a/packages/api/src/middleware/__tests__/tenant.spec.ts b/packages/api/src/middleware/__tests__/tenant.spec.ts index 7247b215c2..0635079831 100644 --- a/packages/api/src/middleware/__tests__/tenant.spec.ts +++ b/packages/api/src/middleware/__tests__/tenant.spec.ts @@ -574,3 +574,22 @@ describe('requirePreAuthTenantContext', () => { expect(next).not.toHaveBeenCalled(); }); }); + +describe('reserved tenant partition', () => { + it('rejects a reserved tenant restored from the authenticated user', () => { + const req = mockReq({ tenantId: '_', role: 'user' }); + const res = mockRes(); + const next = jest.fn(); + tenantContextMiddleware(req, res, next); + expect(res.status).toHaveBeenCalledWith(403); + expect(next).not.toHaveBeenCalled(); + }); + it('rejects the reserved tenant when a request resumes after an async boundary', async () => { + const req = mockReq({ tenantId: '_', role: 'user' }); + const res = mockRes(); + const next = jest.fn(); + await restoreTenantContextFromReq(req, res, next); + expect(res.status).toHaveBeenCalledWith(403); + expect(next).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/middleware/preAuthTenant.spec.ts b/packages/api/src/middleware/preAuthTenant.spec.ts index c92ecbdab4..c7741e8515 100644 --- a/packages/api/src/middleware/preAuthTenant.spec.ts +++ b/packages/api/src/middleware/preAuthTenant.spec.ts @@ -184,3 +184,25 @@ describe('preAuthTenantMiddleware', () => { ); }); }); + +describe('reserved tenant header boundary', () => { + it('rejects the tenantless persistence sentinel before invoking unauthenticated handlers', () => { + const previous = process.env.TRUST_TENANT_HEADER; + process.env.TRUST_TENANT_HEADER = 'true'; + const response = { status: jest.fn().mockReturnThis(), json: jest.fn() }; + const next = jest.fn(); + try { + preAuthTenantMiddleware( + { headers: { 'x-tenant-id': '_' } } as unknown as Request, + response as unknown as Response, + next, + ); + expect(response.status).toHaveBeenCalledWith(400); + expect(response.json).toHaveBeenCalledWith({ error: 'Reserved tenant identifier' }); + expect(next).not.toHaveBeenCalled(); + } finally { + if (previous === undefined) delete process.env.TRUST_TENANT_HEADER; + else process.env.TRUST_TENANT_HEADER = previous; + } + }); +}); diff --git a/packages/api/src/middleware/preAuthTenant.ts b/packages/api/src/middleware/preAuthTenant.ts index 4aa41d332e..845a67810d 100644 --- a/packages/api/src/middleware/preAuthTenant.ts +++ b/packages/api/src/middleware/preAuthTenant.ts @@ -50,6 +50,12 @@ export function preAuthTenantMiddleware(req: Request, res: Response, next: NextF } const rejection = rejectTenantHeader(tenantId); + if (rejection === 'reserved') { + runWithTenantContext(requestContext, () => + res.status(400).json({ error: 'Reserved tenant identifier' }), + ); + return; + } if (rejection === 'system') { runWithTenantContext(requestContext, () => { logger.warn('[preAuthTenant] Rejected __SYSTEM__ sentinel in X-Tenant-Id header', { diff --git a/packages/api/src/middleware/tenant.ts b/packages/api/src/middleware/tenant.ts index 9528deec0c..280c64d64f 100644 --- a/packages/api/src/middleware/tenant.ts +++ b/packages/api/src/middleware/tenant.ts @@ -1,7 +1,13 @@ import { randomUUID } from 'crypto'; import { unlink } from 'fs/promises'; import { isMainThread } from 'worker_threads'; -import { tenantStorage, getTenantId, logger, SYSTEM_TENANT_ID } from '@librechat/data-schemas'; +import { + tenantStorage, + getTenantId, + logger, + SYSTEM_TENANT_ID, + isReservedTenantId, +} from '@librechat/data-schemas'; import type { Response, NextFunction, RequestHandler } from 'express'; import type { TenantContext } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types/http'; @@ -171,11 +177,15 @@ export function tenantContextMiddleware( const context = buildTenantContext(req); const { tenantId } = context; - if (tenantId === SYSTEM_TENANT_ID) { + if (tenantId === SYSTEM_TENANT_ID || isReservedTenantId(tenantId)) { logger.warn('[tenantContextMiddleware] Rejected system tenant for request route', { path: req.path, }); - res.status(403).json({ error: SYSTEM_TENANT_REJECTION_MESSAGE }); + res.status(403).json({ + error: isReservedTenantId(tenantId) + ? 'Reserved tenant identifier' + : SYSTEM_TENANT_REJECTION_MESSAGE, + }); return; } @@ -305,11 +315,17 @@ export function restoreTenantContextFromReq( return; } - if (resolvedTenantId === SYSTEM_TENANT_ID) { + if (resolvedTenantId === SYSTEM_TENANT_ID || isReservedTenantId(resolvedTenantId)) { logger.warn('[restoreTenantContextFromReq] Rejected system tenant for request route', { path: req.path, }); - return rejectRequestWithUploadCleanup(req, res, SYSTEM_TENANT_REJECTION_MESSAGE); + return rejectRequestWithUploadCleanup( + req, + res, + isReservedTenantId(resolvedTenantId) + ? 'Reserved tenant identifier' + : SYSTEM_TENANT_REJECTION_MESSAGE, + ); } const currentContext = tenantStorage.getStore(); diff --git a/packages/api/src/middleware/tenantHeader.spec.ts b/packages/api/src/middleware/tenantHeader.spec.ts new file mode 100644 index 0000000000..9bc5b1ca59 --- /dev/null +++ b/packages/api/src/middleware/tenantHeader.spec.ts @@ -0,0 +1,7 @@ +import { rejectTenantHeader } from './tenantHeader'; + +it('reserves the channel tenantless identifier without banning underscores inside real tenant IDs', () => { + expect(rejectTenantHeader('_')).toBe('reserved'); + expect(rejectTenantHeader('org_1')).toBeUndefined(); + expect(rejectTenantHeader('__SYSTEM__')).toBe('system'); +}); diff --git a/packages/api/src/middleware/tenantHeader.ts b/packages/api/src/middleware/tenantHeader.ts index 8ce3aeca83..45c251058e 100644 --- a/packages/api/src/middleware/tenantHeader.ts +++ b/packages/api/src/middleware/tenantHeader.ts @@ -1,16 +1,19 @@ -import { SYSTEM_TENANT_ID } from '@librechat/data-schemas'; +import { SYSTEM_TENANT_ID, isReservedTenantId } from '@librechat/data-schemas'; const MAX_TENANT_ID_LENGTH = 128; const VALID_TENANT_ID = /^[-a-zA-Z0-9_.]+$/; /** - * Why a trusted `X-Tenant-Id` value names no tenant: the system sentinel, or a value that is + * Why a trusted `X-Tenant-Id` value names no tenant: a reserved persistence/system sentinel, or a value that is * too long or carries characters outside the tenant id alphabet. `undefined` accepts it. */ -export function rejectTenantHeader(tenantId: string): 'system' | 'malformed' | undefined { +export function rejectTenantHeader( + tenantId: string, +): 'system' | 'reserved' | 'malformed' | undefined { if (tenantId === SYSTEM_TENANT_ID) { return 'system'; } + if (isReservedTenantId(tenantId)) return 'reserved'; if (tenantId.length > MAX_TENANT_ID_LENGTH || !VALID_TENANT_ID.test(tenantId)) { return 'malformed'; } diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 8243c7e884..9ca785c935 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -3371,3 +3371,33 @@ describe('directory verification deadline configuration', () => { ); }); }); + +describe('channel storage work limit', () => { + it('preserves the claim default and exposes an operator override through YAML config', () => { + expect(agentsEndpointSchema.parse({ channels: {} }).channels?.claimBatchSize).toBe(25); + expect( + configSchema.parse({ + version: '1.3.17', + endpoints: { agents: { channels: { claimBatchSize: 60 } } }, + }).endpoints?.agents?.channels?.claimBatchSize, + ).toBe(60); + }); + it.each([0, 101, 1.5, '25'])('rejects invalid claim batch %s', (claimBatchSize) => { + expect(agentsEndpointSchema.safeParse({ channels: { claimBatchSize } }).success).toBe(false); + }); +}); + +describe('channel erasure configuration', () => { + it('defaults cleanup batches and exposes a deployment override', () => { + expect(agentsEndpointSchema.parse({ channels: {} }).channels?.erasureBatchSize).toBe(250); + expect( + configSchema.parse({ + version: '1.3.17', + endpoints: { agents: { channels: { erasureBatchSize: 12 } } }, + }).endpoints?.agents?.channels?.erasureBatchSize, + ).toBe(12); + }); + it.each([0, 1001, 1.5, '250'])('rejects invalid erasure batch %s', (erasureBatchSize) => { + expect(agentsEndpointSchema.safeParse({ channels: { erasureBatchSize } }).success).toBe(false); + }); +}); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 6cf3aba026..c572392fe8 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1229,6 +1229,25 @@ const managementApiAuthSchema = z } }); +export const CHANNEL_DELIVERY_PARTS_HARD_MAX = 100; + +/** Storage work per claim transaction; shared by configuration and injected storage defaults. */ +export const channelStorageConfigSchema = z.object({ + erasureBatchSize: z.number().int().min(1).max(1000).default(250), + claimBatchSize: z.number().int().min(1).max(100).default(25), + maxDeliveryParts: z + .number() + .int() + .min(1) + .max(CHANNEL_DELIVERY_PARTS_HARD_MAX) + .default(CHANNEL_DELIVERY_PARTS_HARD_MAX), + maxMetadataPageSize: z.number().int().min(1).max(100).default(100), + projectionYieldMs: z.number().int().min(100).max(60_000).default(1000), + maxPriorSummaryRows: z.number().int().min(1).max(10_000).default(1000), +}); + +export type ChannelStorageConfig = z.infer; + const managementApiSchema = z.object({ auth: managementApiAuthSchema.optional() }).strict(); /** @@ -2260,6 +2279,7 @@ export const agentsEndpointSchema = baseEndpointSchema .optional(), managementApi: managementApiSchema.optional(), remoteApi: remoteApiSchema.optional(), + channels: channelStorageConfigSchema.optional(), /** Human-in-the-loop tool approval policy. Off by default. */ toolApproval: toolApprovalPolicySchema, /** Ask User question behavior; see {@link askUserQuestionConfigSchema}. */ diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 926a56165a..8c1021fe25 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -115,4 +115,6 @@ export * from './approval'; export * from './types/responseReview'; +export * from './types/channelRuns'; + export * from './codeSetup'; diff --git a/packages/data-provider/src/types/channelRuns.ts b/packages/data-provider/src/types/channelRuns.ts new file mode 100644 index 0000000000..4b7a134ff9 --- /dev/null +++ b/packages/data-provider/src/types/channelRuns.ts @@ -0,0 +1,162 @@ +import type { AccountingBasis, ResponseAccounting } from './responseAccounting'; + +/** Verified transport identities are supplied by the gateway, never by Agent output. */ +export interface ChannelRunActor { + teamId: string; + appId: string; + externalUserId: string; +} + +export interface ChannelRunPrincipal { + userId: string; + channelRevision: string; + linkId?: string; + linkRevision?: string; +} + +export interface ChannelRunAdmission { + eventId: string; + externalChannelId: string; + externalThreadKey: string; + externalMessageKey: string; + actor: ChannelRunActor; + input: string; +} + +export type ChannelRunState = + | 'queued' + | 'executing' + | 'requires_action' + | 'delivering' + | 'delivered' + | 'failed' + | 'cancelled' + | 'blocked'; + +export interface ChannelRunFence { + runId: string; + owner: string; + epoch: number; +} + +export interface ChannelRunUsage { + inputTokens: number; + outputTokens: number; + totalTokens: number; + cachedTokens?: number; + reasoningTokens?: number; +} + +export interface ChannelSummaryMetric { + coverage: 'complete' | 'partial' | 'unavailable'; + coveredRuns: number; + /** Present only when every expected run is covered. */ + value?: T; + /** A known subtotal, never presented as the full thread total. */ + knownValue?: T; +} + +/** Current run plus prior attempts in this actor/link revision's lane, not other thread actors. */ +export interface ChannelRunSummary { + version: 1; + scope: 'actor_lane'; + throughSequence: number; + includesCurrent: true; + priorRuns: number; + expectedRuns: number; + observedRuns: number; + costUSD: ChannelSummaryMetric & { basis?: AccountingBasis }; + /** Sum of per-run admission-to-final durations, not the thread's wall-clock span. */ + elapsedMs: ChannelSummaryMetric; + tokens: ChannelSummaryMetric< + Pick + >; + /** Existing authorized trace UI session; never a collector URL or a raw OAuth URL. */ + session?: { url: string }; +} + +/** This is a safe projection, not the Agent transcript or private reasoning. */ +export interface ChannelRunSnapshot { + recovery?: 'execution_state_unavailable'; + responseId?: string; + text: string; + status: 'running' | 'requires_action' | 'completed' | 'failed' | 'cancelled'; + elapsedMs: number; + usage?: ChannelRunUsage; + accounting?: ResponseAccounting; + summary?: ChannelRunSummary; + artifacts?: ChannelArtifact[]; + artifactsUnavailable?: boolean; + activity?: 'working' | 'tool_requested' | 'tool_finished'; + pendingAction?: { + actionId: string; + kind: 'tool_approval' | 'ask_user_question' | 'oauth'; + expiresAt: number; + }; +} + +export interface ChannelArtifact { + artifactId: string; + filename: string; + mediaType: string; + bytes: number; + sha256: string; +} + +/** A deterministic render plan is immutable once its first transport operation starts. */ +export interface ChannelDeliveryPart { + operationId: string; + kind: 'message' | 'artifact'; + required: boolean; + text: string; + /** Provider-defined, versioned JSON presentation; immutable alongside the text fallback. */ + presentation?: { format: string; payload: string }; + /** Opaque server-authorized artifact identity; never an arbitrary URL or disk path. */ + artifactId?: string; +} + +export interface ChannelDeliveryPlan { + revision: string; + rendererVersion: string; + parts: ChannelDeliveryPart[]; +} + +export type ChannelDeliveryOutcome = + | 'planned' + | 'inflight' + | 'confirmed' + | 'retryable' + | 'unknown' + | 'permanent_failure'; + +export interface ChannelDeliveryReceipt { + operationId: string; + outcome: ChannelDeliveryOutcome; + attempts: number; + externalMessageKey?: string; + externalFileId?: string; + retryAt?: number; + errorCode?: string; + upload?: { fileId: string; stage: 'allocated' | 'uploaded' }; +} + +export interface ChannelRunProgress + extends Omit { + revision: number; + rendererVersion: string; + text: string; +} + +export interface ChannelRunView { + runId: string; + channelId: string; + revision: string; + state: ChannelRunState; + admission: ChannelRunAdmission; + sequence: number; + snapshot?: ChannelRunSnapshot; + progress?: ChannelRunProgress; + plan?: ChannelDeliveryPlan; + receipts: ChannelDeliveryReceipt[]; + cancellationRequested: boolean; +} diff --git a/packages/data-schemas/src/config/tenantContext.spec.ts b/packages/data-schemas/src/config/tenantContext.spec.ts index f82c41699c..912b573ea9 100644 --- a/packages/data-schemas/src/config/tenantContext.spec.ts +++ b/packages/data-schemas/src/config/tenantContext.spec.ts @@ -50,3 +50,14 @@ describe('scopedCacheKey', () => { ); }); }); + +describe('reserved channel partition', () => { + it('refuses to create a named-tenant cache namespace for the tenantless sentinel', () => { + expect(() => tenantStorage.run({ tenantId: '_' }, () => scopedCacheKey('private'))).toThrow( + 'Reserved tenant identifier', + ); + expect(tenantStorage.run({ tenantId: 'org_1' }, () => scopedCacheKey('private'))).toBe( + 'private:org_1', + ); + }); +}); diff --git a/packages/data-schemas/src/config/tenantContext.ts b/packages/data-schemas/src/config/tenantContext.ts index 08a1c2b214..aed72851b2 100644 --- a/packages/data-schemas/src/config/tenantContext.ts +++ b/packages/data-schemas/src/config/tenantContext.ts @@ -11,6 +11,11 @@ export interface TenantContext { /** Sentinel value for deliberate cross-tenant system operations */ export const SYSTEM_TENANT_ID = '__SYSTEM__'; +/** Reserved for persisted tenantless channel partitions; never an authenticated tenant. */ +export function isReservedTenantId(tenantId: string | undefined): boolean { + return tenantId === '_'; +} + /** * AsyncLocalStorage instance for propagating tenant context. * Callbacks passed to `tenantStorage.run()` must be `async` for the context to propagate @@ -21,7 +26,9 @@ export const tenantStorage: AsyncLocalStorage = /** Returns the current tenant ID from async context, or undefined if none is set */ export function getTenantId(): string | undefined { - return tenantStorage.getStore()?.tenantId; + const tenantId = tenantStorage.getStore()?.tenantId; + if (isReservedTenantId(tenantId)) throw new TypeError('Reserved tenant identifier'); + return tenantId; } /** Returns the current user ID from async context, or undefined if none is set */ diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 1fec5d0839..20a95d2f0d 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -29,6 +29,10 @@ export type { } from './methods/response'; export { createMethods, + createChannelRunStorage, + createChannelSummaryStorage, + CHANNEL_SUMMARY_MAX_PRIOR_ROWS, + ChannelRunConflict, createSkillDeletionPersistenceAdapter, AGENT_OWNER_CONTACT_RESOLVED_FIELD, AgentSortCursorError, @@ -133,6 +137,7 @@ export { runAsSystem, scopedCacheKey, SYSTEM_TENANT_ID, + isReservedTenantId, } from './config/tenantContext'; export type { TenantContext } from './config/tenantContext'; export { diff --git a/packages/data-schemas/src/methods/channelErasure.ts b/packages/data-schemas/src/methods/channelErasure.ts new file mode 100644 index 0000000000..aa9069496a --- /dev/null +++ b/packages/data-schemas/src/methods/channelErasure.ts @@ -0,0 +1,41 @@ +import { channelStorageConfigSchema } from 'librechat-data-provider'; +import type { ChannelStorageConfig } from 'librechat-data-provider'; +import type { ResponseOwner } from '~/models/response'; +import { createChannelRunModel, createChannelLaneModel } from '~/models/channelRun'; + +/** Validate operation configuration before the caller closes irreversible admission fences. */ +export function resolveChannelErasureConfig( + options: Partial = {}, +): ChannelStorageConfig { + return channelStorageConfigSchema.parse(options); +} + +/** The caller closes the response owner admission anchor before entering this cascade. */ +export async function eraseChannelPersistenceForUser( + mongoose: typeof import('mongoose'), + owner: ResponseOwner, + options: Pick, 'erasureBatchSize'> = {}, +): Promise { + const { erasureBatchSize } = resolveChannelErasureConfig(options); + const tenantId = owner.tenantId || '_'; + const runs = createChannelRunModel(mongoose); + const lanes = createChannelLaneModel(mongoose); + const filter = { tenantId, 'principal.userId': owner.userId }; + // New lanes have an indexed owner. Older rows are recovered from the still-retained runs. + await lanes.deleteMany({ tenantId, userId: owner.userId }); + const cursor = runs.find(filter).select('laneId').lean().cursor(); + const batch = new Set(); + try { + for await (const run of cursor) { + batch.add(run.laneId); + if (batch.size === erasureBatchSize) { + await lanes.deleteMany({ tenantId, laneId: { $in: [...batch] } }); + batch.clear(); + } + } + if (batch.size) await lanes.deleteMany({ tenantId, laneId: { $in: [...batch] } }); + } finally { + await cursor.close(); + } + await runs.deleteMany(filter); +} diff --git a/packages/data-schemas/src/methods/channelIndexes.spec.ts b/packages/data-schemas/src/methods/channelIndexes.spec.ts new file mode 100644 index 0000000000..d5d130e610 --- /dev/null +++ b/packages/data-schemas/src/methods/channelIndexes.spec.ts @@ -0,0 +1,37 @@ +import { createChannelIndexGate } from './channelIndexes'; + +it('serializes required index builds and shares the readiness promise across concurrent admissions', async () => { + const order: string[] = []; + const first = { + modelName: 'First', + createIndexes: jest.fn(async () => { + order.push('first'); + }), + }; + const second = { + modelName: 'Second', + createIndexes: jest.fn(async () => { + order.push('second'); + }), + }; + const ready = createChannelIndexGate(() => [first, second]); + await Promise.all([ready(), ready(), ready()]); + await ready(); + expect(order).toEqual(['first', 'second']); + expect(first.createIndexes).toHaveBeenCalledTimes(1); + expect(second.createIndexes).toHaveBeenCalledTimes(1); +}); + +it('fails closed on index failure and permits a later readiness retry', async () => { + const model = { + modelName: 'ChannelRun', + createIndexes: jest + .fn() + .mockRejectedValueOnce(new Error('index rejected')) + .mockResolvedValue(undefined), + }; + const ready = createChannelIndexGate(() => [model]); + await expect(ready()).rejects.toThrow('index rejected'); + await expect(ready()).resolves.toBeUndefined(); + expect(model.createIndexes).toHaveBeenCalledTimes(2); +}); diff --git a/packages/data-schemas/src/methods/channelIndexes.ts b/packages/data-schemas/src/methods/channelIndexes.ts new file mode 100644 index 0000000000..c0d7a3dd85 --- /dev/null +++ b/packages/data-schemas/src/methods/channelIndexes.ts @@ -0,0 +1,19 @@ +import { createIndexesWithRetry } from '~/utils/retry'; + +/** Unique indexes are correctness fences, even when automatic indexing is disabled. */ +export function createChannelIndexGate( + models: () => Array[0]>, + afterReady?: () => Promise, +): () => Promise { + let pending: Promise | undefined; + return () => { + pending ??= (async () => { + for (const model of models()) await createIndexesWithRetry(model); + await afterReady?.(); + })().catch((error: unknown) => { + pending = undefined; + throw error; + }); + return pending; + }; +} diff --git a/packages/data-schemas/src/methods/channelOwner.ts b/packages/data-schemas/src/methods/channelOwner.ts new file mode 100644 index 0000000000..2ccd396d99 --- /dev/null +++ b/packages/data-schemas/src/methods/channelOwner.ts @@ -0,0 +1,29 @@ +import type { ClientSession } from 'mongoose'; +import type { ResponseOwner } from '~/models/response'; +import { responseScopeKeys, initializeResponseScopes } from './responseScope'; +import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext'; +import { createResponseScopeModel } from '~/models/responseScope'; + +/** All private channel persistence joins the same account-erasure admission fence. */ +export async function prepareChannelOwnerAdmission( + mongoose: typeof import('mongoose'), + owner: ResponseOwner, +): Promise<(session: ClientSession) => Promise> { + const current = getTenantId() || undefined; + if ( + !owner.userId || + owner.tenantId === SYSTEM_TENANT_ID || + (current && current !== SYSTEM_TENANT_ID && current !== (owner.tenantId || undefined)) + ) + throw new TypeError('Channel owner does not match authenticated scope'); + const [key] = responseScopeKeys(owner, ''); + await initializeResponseScopes(mongoose, [key]); + return async (session) => { + const admitted = await createResponseScopeModel(mongoose).updateOne( + { _id: key, closed: false }, + { $inc: { revision: 1 } }, + { session }, + ); + return admitted.matchedCount === 1; + }; +} diff --git a/packages/data-schemas/src/methods/channelProtocol.spec.ts b/packages/data-schemas/src/methods/channelProtocol.spec.ts new file mode 100644 index 0000000000..ae935b696f --- /dev/null +++ b/packages/data-schemas/src/methods/channelProtocol.spec.ts @@ -0,0 +1,133 @@ +import type { ChannelDeliveryOutcome, ChannelRunSnapshot } from 'librechat-data-provider'; +import type { ChannelRunRecord } from '~/models/channelRun'; +import { deliveryDisposition, snapshotPatch } from './channelProtocol'; + +function run(): ChannelRunRecord { + return { + tenantId: 'tenant', + channelId: 'channel', + agentId: 'agent', + laneId: 'lane', + runId: 'run', + principal: { userId: 'user', channelRevision: 'revision' }, + payloadHash: 'hash', + sequence: 1, + admission: { + eventId: 'event', + externalChannelId: 'C', + externalThreadKey: 'C:thread', + externalMessageKey: 'C:event', + actor: { teamId: 'team', appId: 'app', externalUserId: 'actor' }, + input: 'question', + }, + receivedAt: 1, + state: 'delivering', + leaseEpoch: 1, + leaseExpiresAt: 1000, + cancellationRequested: false, + execution: { + requestId: 'run', + conversationId: 'conversation', + parentMessageId: 'parent', + responseId: 'response', + }, + snapshot: { status: 'completed', responseId: 'response', text: 'answer', elapsedMs: 1 }, + plan: { + revision: 'plan', + rendererVersion: 'test', + parts: [ + { operationId: 'required', kind: 'message', required: true, text: 'answer' }, + { operationId: 'optional', kind: 'message', required: false, text: 'footer' }, + ], + }, + receipts: [ + { + operationId: 'required', + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C:required', + }, + { + operationId: 'optional', + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C:optional', + }, + ], + }; +} + +const outcomes: ChannelDeliveryOutcome[] = [ + 'planned', + 'inflight', + 'confirmed', + 'retryable', + 'unknown', + 'permanent_failure', +]; +const deliveryCases = outcomes.flatMap((required) => + outcomes.map((optional) => ({ required, optional })), +); +it.each(deliveryCases)( + 'finalizes required=$required and optional=$optional according to delivery policy', + ({ required, optional }) => { + const current = run(); + current.receipts[0].outcome = required; + current.receipts[1].outcome = optional; + const canFinish = + ![required, optional].includes('inflight') && + ['confirmed', 'permanent_failure'].includes(required); + if (!canFinish) expect(() => deliveryDisposition(current, 100)).toThrow('delivery_incomplete'); + else + expect(deliveryDisposition(current, 100)).toBe( + required === 'permanent_failure' ? 'failed' : 'delivered', + ); + }, +); + +const statuses: ChannelRunSnapshot['status'][] = [ + 'running', + 'requires_action', + 'completed', + 'failed', + 'cancelled', +]; +const snapshotCases = statuses.flatMap((previous) => statuses.map((next) => ({ previous, next }))); +it.each(snapshotCases)( + 'projects $previous to $next without reopening terminal work', + ({ previous, next }) => { + const current = run(); + delete current.plan; + current.snapshot = { ...current.snapshot!, status: previous }; + const phases: Record = { + running: 'executing', + requires_action: 'requires_action', + completed: 'delivering', + failed: 'delivering', + cancelled: 'delivering', + }; + current.state = phases[previous]; + const incoming = { ...current.snapshot, status: next }; + if (['completed', 'failed', 'cancelled'].includes(previous) && previous !== next) + expect(() => snapshotPatch(current, incoming)).toThrow('invalid_transition'); + else expect(snapshotPatch(current, incoming).state).toBe(phases[next]); + }, +); + +it('does not finalize while a progress operation is still active', () => { + const current = run(); + current.progress = { + revision: 1, + rendererVersion: 'test', + text: 'progress', + outcome: 'inflight', + attempts: 1, + }; + expect(() => deliveryDisposition(current, 100)).toThrow('delivery_incomplete'); +}); + +it('rejects duplicate receipt identities instead of treating an incomplete plan as confirmed', () => { + const current = run(); + current.receipts[1] = current.receipts[0]; + expect(() => deliveryDisposition(current, 100)).toThrow('invalid_transition'); +}); diff --git a/packages/data-schemas/src/methods/channelProtocol.ts b/packages/data-schemas/src/methods/channelProtocol.ts new file mode 100644 index 0000000000..6bacd5d293 --- /dev/null +++ b/packages/data-schemas/src/methods/channelProtocol.ts @@ -0,0 +1,240 @@ +import { createHash } from 'node:crypto'; +import type { + ChannelRunAdmission, + ChannelRunPrincipal, + ChannelRunSnapshot, + ChannelDeliveryPlan, + ChannelDeliveryReceipt, +} from 'librechat-data-provider'; +import type { ChannelRunRecord } from '~/models/channelRun'; + +export interface ChannelLaneScope { + tenantId: string; + channelId: string; + agentId: string; + principal: ChannelRunPrincipal; + admission: Pick; +} + +export interface ChannelRunEnrollment extends ChannelLaneScope { + admission: ChannelRunAdmission; +} + +export class ChannelRunConflict extends Error { + constructor( + readonly code: 'event_conflict' | 'stale_fence' | 'invalid_transition' | 'delivery_incomplete', + ) { + super(code); + this.name = 'ChannelRunConflict'; + } +} + +export const digest = (value: object): string => + createHash('sha256') + .update( + JSON.stringify(value, (_key, item: unknown) => + item != null && typeof item === 'object' && !Array.isArray(item) + ? Object.fromEntries( + Object.entries(item).sort(([a], [b]) => { + if (a === b) return 0; + return a < b ? -1 : 1; + }), + ) + : item, + ), + ) + .digest('hex'); +export function laneKey(scope: ChannelLaneScope): string { + return digest({ + tenantId: scope.tenantId, + channelId: scope.channelId, + agentId: scope.agentId, + thread: scope.admission.externalThreadKey, + externalChannelId: scope.admission.externalChannelId, + actor: scope.admission.actor, + userId: scope.principal.userId, + revision: scope.principal.channelRevision, + linkRevision: scope.principal.linkRevision ?? null, + linkId: scope.principal.linkId ?? null, + }); +} +export const activeStates: string[] = [ + 'queued', + 'executing', + 'requires_action', + 'delivering', + 'blocked', +]; + +/** A projection cannot grant a different response authority to the next actor-lane turn. */ +export function assertSnapshotBinding(run: ChannelRunRecord, snapshot: ChannelRunSnapshot): void { + if ( + (snapshot.status === 'completed' && !snapshot.responseId) || + (snapshot.responseId != null && snapshot.responseId !== run.execution?.responseId) + ) + throw new ChannelRunConflict('invalid_transition'); +} + +/** Confirm the provider identity appropriate to the immutable plan, on every confirmation path. */ +export function assertDeliveryEvidence( + run: ChannelRunRecord, + receipt: ChannelDeliveryReceipt, + previous: ChannelDeliveryReceipt | undefined, +): void { + const part = run.plan?.parts.find((item) => item.operationId === receipt.operationId); + if ( + !part || + (part.kind === 'artifact' && + (!part.artifactId || + !run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))) || + !previous || + (previous.externalMessageKey != null && + receipt.externalMessageKey != null && + previous.externalMessageKey !== receipt.externalMessageKey) || + (previous.externalFileId != null && + receipt.externalFileId != null && + previous.externalFileId !== receipt.externalFileId) || + (receipt.upload != null && digest(receipt.upload) !== digest(previous.upload ?? {})) || + (receipt.externalFileId != null && + previous.upload != null && + previous.upload.fileId !== receipt.externalFileId) || + (receipt.outcome === 'confirmed' && + ((part.kind === 'message' && !receipt.externalMessageKey) || + (part.kind === 'artifact' && + (!receipt.externalFileId || + (previous.upload != null && previous.upload.fileId !== receipt.externalFileId))))) + ) + throw new ChannelRunConflict('invalid_transition'); +} + +export function leaseDeadline(now: number, leaseMs: number): number { + if ( + !Number.isSafeInteger(now) || + !Number.isSafeInteger(leaseMs) || + leaseMs <= 0 || + !Number.isSafeInteger(now + leaseMs) + ) + throw new Error('Invalid channel lease'); + return now + leaseMs; +} + +/** Evidence is verified by the transport; storage binds it to the exact uncertain attempt. */ +export function assertReconciliation( + previous: Omit | undefined, + receipt: Pick, +): void { + if ( + !previous || + !['unknown', 'confirmed'].includes(previous.outcome) || + previous.attempts !== receipt.attempts || + (!receipt.externalMessageKey && !receipt.externalFileId) || + (previous.externalMessageKey && previous.externalMessageKey !== receipt.externalMessageKey) || + (previous.externalFileId && previous.externalFileId !== receipt.externalFileId) || + (previous.upload && previous.upload.fileId !== receipt.externalFileId) || + (previous.outcome === 'confirmed' && + (previous.externalMessageKey !== receipt.externalMessageKey || + previous.externalFileId !== receipt.externalFileId)) + ) + throw new ChannelRunConflict('invalid_transition'); +} + +const terminalSnapshots = ['completed', 'failed', 'cancelled']; + +export function snapshotPatch( + run: ChannelRunRecord, + snapshot: ChannelRunSnapshot, +): Pick { + assertSnapshotBinding(run, snapshot); + if ( + run.snapshot && + terminalSnapshots.includes(run.snapshot.status) && + (run.snapshot.status !== snapshot.status || + run.snapshot.responseId !== snapshot.responseId || + run.snapshot.text !== snapshot.text) + ) + throw new ChannelRunConflict('invalid_transition'); + if (run.state === 'delivering' && !terminalSnapshots.includes(snapshot.status)) + throw new ChannelRunConflict('invalid_transition'); + let state: ChannelRunRecord['state'] = 'delivering'; + if (snapshot.status === 'running') state = 'executing'; + else if (snapshot.status === 'requires_action') state = 'requires_action'; + const patch: Pick = { + state, + snapshot: + snapshot.summary == null && run.snapshot?.summary + ? { ...snapshot, summary: run.snapshot.summary } + : snapshot, + }; + if (run.plan && digest(patch.snapshot ?? {}) !== digest(run.snapshot ?? {})) + throw new ChannelRunConflict('invalid_transition'); + return patch; +} + +export function assertPlan( + run: ChannelRunRecord, + plan: ChannelDeliveryPlan, + maxParts: number, +): void { + if ( + !run.snapshot || + !terminalSnapshots.includes(run.snapshot.status) || + run.state !== 'delivering' || + !plan.parts.length || + plan.parts.length > maxParts || + new Set(plan.parts.map((part) => part.operationId)).size !== plan.parts.length || + !plan.parts.some((part) => part.required) || + plan.parts.some( + (part) => + !part.operationId || + !['message', 'artifact'].includes(part.kind) || + (part.kind === 'artifact' && + (!part.artifactId || + !run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))), + ) + ) + throw new ChannelRunConflict('invalid_transition'); + assertSnapshotBinding(run, run.snapshot); +} + +export function hasPermanentDeliveryFailure(run: ChannelRunRecord): boolean { + return ( + run.plan?.parts.some( + (part) => + part.required && + run.receipts.some( + (receipt) => + receipt.operationId === part.operationId && receipt.outcome === 'permanent_failure', + ), + ) ?? false + ); +} + +/** A permanent required failure is terminal even though successful delivery is impossible. + * Unknown receipts never become eligible for resend. Optional unknown effects remain best effort. */ +export function deliveryDisposition( + run: ChannelRunRecord, + maxParts: number, +): 'delivered' | 'failed' | 'cancelled' { + if (!run.plan || !run.snapshot) throw new ChannelRunConflict('delivery_incomplete'); + assertPlan(run, run.plan, maxParts); + if ( + run.receipts.length !== run.plan.parts.length || + new Set(run.receipts.map((receipt) => receipt.operationId)).size !== run.receipts.length + ) + throw new ChannelRunConflict('invalid_transition'); + for (const receipt of run.receipts) assertDeliveryEvidence(run, receipt, receipt); + if ( + run.receipts.some((receipt) => receipt.outcome === 'inflight') || + run.progress?.outcome === 'inflight' + ) + throw new ChannelRunConflict('delivery_incomplete'); + const required = run.plan.parts + .filter((part) => part.required) + .map((part) => run.receipts.find((receipt) => receipt.operationId === part.operationId)); + if (hasPermanentDeliveryFailure(run)) return 'failed'; + if (required.some((receipt) => receipt?.outcome !== 'confirmed')) + throw new ChannelRunConflict('delivery_incomplete'); + if (run.snapshot.status === 'completed') return 'delivered'; + if (run.snapshot.status === 'cancelled') return 'cancelled'; + return 'failed'; +} diff --git a/packages/data-schemas/src/methods/channelRun.spec.ts b/packages/data-schemas/src/methods/channelRun.spec.ts new file mode 100644 index 0000000000..1ba2b3b475 --- /dev/null +++ b/packages/data-schemas/src/methods/channelRun.spec.ts @@ -0,0 +1,1293 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import type { ChannelRunSummary } from 'librechat-data-provider'; +import type { ChannelRunRecord } from '~/models/channelRun'; +import { tenantStorage, runAsSystem } from '~/config/tenantContext'; +import { createChannelRunStorage } from './channelRun'; +import { waitForPause } from './test-helpers'; +import { createUserMethods } from './user'; +import { createModels } from '~/models'; + +let mongo: MongoMemoryReplSet; +let storage: ReturnType; +const tenantId = 'channel-storage-test'; +const scope = (work: () => Promise) => tenantStorage.run({ tenantId }, work); +const fence = (run: ChannelRunRecord) => ({ + runId: run.runId, + owner: run.leaseOwner!, + epoch: run.leaseEpoch, +}); +const event = (eventId = 'event', actorId = 'actor') => ({ + tenantId, + channelId: 'channel', + agentId: 'agent', + principal: { userId: actorId, channelRevision: 'revision' }, + admission: { + eventId, + externalChannelId: 'C1', + externalThreadKey: 'C1:1', + externalMessageKey: `C1:${eventId}`, + actor: { teamId: 'T1', appId: 'A1', externalUserId: actorId }, + input: 'hello', + }, +}); + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createChannelRunStorage(mongoose); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.ChannelRun.deleteMany({}); + await mongoose.models.ChannelLane.deleteMany({}); + await mongoose.models.StoredResponse.deleteMany({}); + }); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +it('admits concurrent retries as one event and one lane sequence', () => + scope(async () => { + const attempts = await Promise.all( + Array.from({ length: 5 }, () => storage.admit(event(), 100)), + ); + expect(new Set(attempts.map(({ run }) => run.runId)).size).toBe(1); + expect(attempts.filter(({ duplicate }) => !duplicate)).toHaveLength(1); + expect(await mongoose.models.ChannelRun.countDocuments({ tenantId })).toBe(1); + expect(await mongoose.models.ChannelLane.findOne({ tenantId }).lean()).toMatchObject({ + nextSequence: 1, + }); + })); + +it('rejects changed payloads that reuse an admitted event identity', () => + scope(async () => { + await storage.admit(event(), 100); + const changed = event(); + changed.admission.input = 'different'; + await expect(storage.admit(changed, 101)).rejects.toMatchObject({ code: 'event_conflict' }); + })); + +it('orders one actor while allowing another actor to make progress', () => + scope(async () => { + const first = await storage.admit(event(), 100); + await storage.admit(event('next'), 101); + const other = await storage.admit(event('other', 'other-actor'), 102); + expect((await storage.claim(tenantId, 'channel', 'worker', 103, 1000))?.runId).toBe( + first.run.runId, + ); + expect((await storage.claim(tenantId, 'channel', 'other-worker', 104, 1000))?.runId).toBe( + other.run.runId, + ); + expect(await storage.claim(tenantId, 'channel', 'third-worker', 105, 1000)).toBeNull(); + })); + +it('rejects a superseded worker after its lease is reclaimed', () => + scope(async () => { + await storage.admit(event(), 100); + const first = (await storage.claim(tenantId, 'channel', 'worker', 101, 10))!; + await storage.prepareExecution(tenantId, fence(first), 102); + const replacement = (await storage.claim(tenantId, 'channel', 'replacement', 112, 1000))!; + expect(replacement.leaseEpoch).toBeGreaterThan(first.leaseEpoch); + await expect( + storage.saveSnapshot( + tenantId, + fence(first), + { status: 'completed', text: 'answer', elapsedMs: 1 }, + 113, + ), + ).rejects.toMatchObject({ code: 'stale_fence' }); + })); + +it('holds the lane until every required delivery has been confirmed', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await bindTestResponse(run); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'completed', text: 'answer', responseId: 'response', elapsedMs: 1 }, + 103, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'plan', + rendererVersion: 'test', + parts: [{ operationId: 'message', kind: 'message', required: true, text: 'answer' }], + }, + 104, + ); + await expect(storage.commitDelivery(tenantId, fence(run), 105)).rejects.toMatchObject({ + code: 'delivery_incomplete', + }); + expect(await mongoose.models.ChannelLane.findOne({ tenantId }).lean()).toMatchObject({ + activeRunId: run.runId, + }); + })); + +it('isolates actor lanes across distinct personal links with the same revision', () => + scope(async () => { + const first = event('link-one'); + const second = event('link-two'); + const a = await storage.admit( + { ...first, principal: { ...first.principal, linkId: 'one', linkRevision: '1' } }, + 100, + ); + const b = await storage.admit( + { ...second, principal: { ...second.principal, linkId: 'two', linkRevision: '1' } }, + 101, + ); + expect(a.run.laneId).not.toBe(b.run.laneId); + const claimedA = (await storage.claim(tenantId, 'channel', 'worker-a', 102, 1000))!; + const claimedB = (await storage.claim(tenantId, 'channel', 'worker-b', 103, 1000))!; + const preparedA = await storage.prepareExecution(tenantId, fence(claimedA), 104); + const preparedB = await storage.prepareExecution(tenantId, fence(claimedB), 104); + expect(preparedA.execution?.conversationId).not.toBe(preparedB.execution?.conversationId); + expect(preparedA.execution?.parentMessageId).toBe('00000000-0000-0000-0000-000000000000'); + expect(preparedB.execution?.parentMessageId).toBe('00000000-0000-0000-0000-000000000000'); + })); + +it('deduplicates equivalent nested event fields regardless of object construction order', () => + scope(async () => { + const input = event(); + const first = await storage.admit(input, 100); + const { admission } = input; + const replay = await storage.admit( + { + ...input, + admission: { + input: admission.input, + actor: { + externalUserId: admission.actor.externalUserId, + appId: admission.actor.appId, + teamId: admission.actor.teamId, + }, + externalMessageKey: admission.externalMessageKey, + externalThreadKey: admission.externalThreadKey, + externalChannelId: admission.externalChannelId, + eventId: admission.eventId, + }, + }, + 101, + ); + expect(replay.duplicate).toBe(true); + expect(replay.run.runId).toBe(first.run.runId); + expect(await mongoose.models.ChannelRun.countDocuments({ tenantId })).toBe(1); + })); + +it('atomically rejects a delayed cancellation after the observed run is reclaimed', () => + scope(async () => { + await storage.admit(event(), 100); + const first = (await storage.claim(tenantId, 'channel', 'worker', 101, 10))!; + const observed = await storage.readActiveThread(event(), 105); + expect(observed?.runId).toBe(first.runId); + const replacement = (await storage.claim(tenantId, 'channel', 'replacement', 112, 1000))!; + expect( + await storage.requestCancel(event(), first.runId, { + epoch: first.leaseEpoch, + before: 105, + }), + ).toBe(false); + expect((await storage.read(tenantId, 'channel', first.runId))?.cancellationRequested).toBe( + false, + ); + expect( + await storage.requestCancel(event(), first.runId, { + epoch: replacement.leaseEpoch, + before: 105, + }), + ).toBe(false); + expect( + await storage.requestCancel(event('event', 'intruder'), first.runId, { + epoch: replacement.leaseEpoch, + before: 113, + }), + ).toBe(false); + expect( + await storage.requestCancel(event(), first.runId, { + epoch: replacement.leaseEpoch, + before: 113, + }), + ).toBe(true); + })); + +async function plannedRun() { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 10))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await bindTestResponse(run); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'completed', text: 'answer', responseId: 'response', elapsedMs: 1 }, + 103, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'plan', + rendererVersion: 'test', + parts: [{ operationId: 'message', kind: 'message', required: true, text: 'answer' }], + }, + 104, + ); + return run; +} + +it('reconciles a verified uncertain delivery with a current fence, without blind resending', () => + scope(async () => { + const run = await plannedRun(); + await storage.beginDelivery(tenantId, fence(run), 'message', 105); + const replacement = (await storage.claim(tenantId, 'channel', 'replacement', 112, 1000))!; + expect(replacement.receipts[0]).toMatchObject({ outcome: 'unknown', attempts: 1 }); + await expect( + storage.beginDelivery(tenantId, fence(replacement), 'message', 113), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + const evidence = { + operationId: 'message', + attempts: 1, + outcome: 'confirmed' as const, + externalMessageKey: 'C1:verified', + }; + await expect( + storage.reconcileDelivery(tenantId, fence(run), evidence, 113), + ).rejects.toMatchObject({ code: 'stale_fence' }); + await expect( + storage.reconcileDelivery(tenantId, fence(replacement), { ...evidence, attempts: 2 }, 113), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await expect( + storage.reconcileDelivery( + tenantId, + fence(replacement), + { ...evidence, externalMessageKey: undefined }, + 113, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.reconcileDelivery(tenantId, fence(replacement), evidence, 113); + await storage.reconcileDelivery(tenantId, fence(replacement), evidence, 114); + await expect( + storage.reconcileDelivery( + tenantId, + fence(replacement), + { ...evidence, externalMessageKey: 'C1:other' }, + 114, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + expect((await storage.commitDelivery(tenantId, fence(replacement), 115)).state).toBe( + 'delivered', + ); + await storage.admit(event('next'), 116); + expect( + (await storage.claim(tenantId, 'channel', 'next-worker', 117, 1000))?.admission.eventId, + ).toBe('next'); + })); + +it('rejects reconciliation of an operation that has never been sent', () => + scope(async () => { + const run = await plannedRun(); + await expect( + storage.reconcileDelivery( + tenantId, + fence(run), + { + operationId: 'message', + attempts: 0, + outcome: 'confirmed', + externalMessageKey: 'fabricated', + }, + 105, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('reconciles uncertain progress with the same attempt and immutable provider message', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 10))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.planProgress( + tenantId, + fence(run), + { revision: 1, rendererVersion: 'test', text: 'working' }, + 103, + ); + await storage.beginProgress(tenantId, fence(run), 1, 104); + const replacement = (await storage.claim(tenantId, 'channel', 'replacement', 112, 1000))!; + const receipt = { revision: 1, attempts: 1, externalMessageKey: 'C1:progress' }; + await expect( + storage.reconcileProgress(tenantId, fence(run), receipt, 113), + ).rejects.toMatchObject({ code: 'stale_fence' }); + await expect( + storage.reconcileProgress(tenantId, fence(replacement), { ...receipt, revision: 2 }, 113), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.reconcileProgress(tenantId, fence(replacement), receipt, 113); + await storage.reconcileProgress(tenantId, fence(replacement), receipt, 114); + await expect( + storage.reconcileProgress( + tenantId, + fence(replacement), + { ...receipt, externalMessageKey: 'C1:other' }, + 114, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.planProgress( + tenantId, + fence(replacement), + { revision: 2, rendererVersion: 'test', text: 'still working' }, + 115, + ); + })); + +it('replays response binding after the authoritative parent differs from the original request', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await mongoose.models.StoredResponse.create({ + tenantId, + userId: 'actor', + responseId: 'bound-response', + requestKey: `channel:${run.runId}`, + fingerprint: 'request', + agentId: 'agent', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'hello', + background: true, + store: true, + }), + conversationId: 'canonical-conversation', + parentMessageId: 'canonical-parent', + status: 'queued', + revision: 0, + }); + const first = await storage.bindResponse(tenantId, fence(run), 'bound-response', 103); + const replay = await storage.bindResponse(tenantId, fence(run), 'bound-response', 104); + expect(first.execution?.parentMessageId).toBe('canonical-parent'); + expect(replay.execution).toEqual(first.execution); + await expect( + storage.bindResponse(tenantId, fence(run), 'different-response', 105), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('compares immutable delivery plans semantically while preserving part order', () => + scope(async () => { + const run = await plannedRun(); + await storage.beginDelivery(tenantId, fence(run), 'message', 105); + await expect( + storage.savePlan( + tenantId, + fence(run), + { + parts: [{ text: 'answer', required: true, kind: 'message', operationId: 'message' }], + rendererVersion: 'test', + revision: 'plan', + }, + 106, + ), + ).resolves.toMatchObject({ plan: { revision: 'plan' } }); + await expect( + storage.savePlan( + tenantId, + fence(run), + { + parts: [{ text: 'changed', required: true, kind: 'message', operationId: 'message' }], + rendererVersion: 'test', + revision: 'plan', + }, + 107, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('seeks active predecessors without scanning completed lane history', () => + scope(async () => { + const { run } = await storage.admit(event(), 100); + await mongoose.models.ChannelRun.updateOne( + { runId: run.runId }, + { $set: { state: 'delivered' } }, + ); + await mongoose.models.ChannelRun.insertMany( + Array.from({ length: 200 }, (_, i) => ({ + ...event(`history-${i}`), + laneId: run.laneId, + payloadHash: 'historical', + sequence: i + 2, + runId: `history-${i}`, + state: 'delivered', + receivedAt: 0, + leaseEpoch: 0, + leaseExpiresAt: 0, + receipts: [], + cancellationRequested: false, + })), + ); + const plan = await mongoose.models.ChannelRun.collection + .find({ + tenantId, + laneId: run.laneId, + state: { $in: ['queued', 'executing', 'requires_action', 'delivering', 'blocked'] }, + sequence: { $lt: 1000 }, + }) + .hint('channel_run_active_predecessor') + .explain('executionStats'); + expect(plan.executionStats.nReturned).toBe(0); + expect(plan.executionStats.totalKeysExamined).toBeLessThan(10); + })); + +it('does not replay an admitted event through a different principal or linked account', () => + scope(async () => { + const input = event(); + await storage.admit(input, 100); + for (const principal of [ + { ...input.principal, userId: 'another-owner' }, + { ...input.principal, linkId: 'another-link' }, + { ...input.principal, channelRevision: 'new-revision' }, + ]) + await expect(storage.admit({ ...input, principal }, 101)).rejects.toMatchObject({ + code: 'event_conflict', + }); + await expect(storage.admit({ ...input, agentId: 'another-agent' }, 101)).rejects.toMatchObject({ + code: 'event_conflict', + }); + })); + +it('erases channel request/output and legacy lane identity before deleting its account', () => + scope(async () => { + const user = await mongoose.models.User.create({ + email: 'channel-erasure@example.com', + provider: 'local', + tenantId, + }); + const userId = String(user._id); + const input = { ...event(), principal: { ...event().principal, userId } }; + const own = await storage.admit(input, 100); + const other = await storage.admit(event('other', 'other-owner'), 100); + await mongoose.models.ChannelLane.updateOne( + { laneId: own.run.laneId }, + { $unset: { userId: 1 } }, + ); + await createUserMethods(mongoose).deleteUserById(userId); + expect(await storage.read(tenantId, 'channel', own.run.runId)).toBeNull(); + expect(await mongoose.models.ChannelLane.exists({ laneId: own.run.laneId })).toBeNull(); + expect(await storage.read(tenantId, 'channel', other.run.runId)).not.toBeNull(); + await expect(storage.admit(input, 101)).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('serializes account erasure with an already inserting channel event', () => + scope(async () => { + const user = await mongoose.models.User.create({ + email: 'channel-erasure-race@example.com', + provider: 'local', + tenantId, + }); + const userId = String(user._id); + let entered!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const collection = mongoose.models.ChannelRun.collection; + const insert = collection.insertOne.bind(collection); + const paused = jest.spyOn(collection, 'insertOne').mockImplementation(async (...args) => { + entered(); + await gate; + return insert(...args); + }); + const pending = storage.admit({ ...event(), principal: { ...event().principal, userId } }, 100); + await waitForPause(started, pending); + let erased = false; + const deletion = createUserMethods(mongoose) + .deleteUserById(userId) + .then((result) => { + erased = true; + return result; + }); + const drained = Promise.allSettled([pending, deletion]); + try { + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(erased).toBe(false); + } finally { + release(); + paused.mockRestore(); + await drained; + } + await pending; + await deletion; + expect(await mongoose.models.ChannelRun.exists({ 'principal.userId': userId })).toBeNull(); + expect(await mongoose.models.ChannelLane.exists({ userId })).toBeNull(); + })); + +async function bindTestResponse(run: ChannelRunRecord, responseId = 'response') { + await mongoose.models.StoredResponse.create({ + tenantId: run.tenantId === '_' ? undefined : run.tenantId, + sourceKeyId: (await storage.read(run.tenantId, run.channelId, run.runId))?.execution?.apiKeyId, + userId: run.principal.userId, + responseId, + requestKey: `channel:${run.runId}`, + fingerprint: 'request', + agentId: run.agentId, + requestJSON: JSON.stringify({ + model: run.agentId, + input: run.admission.input, + background: true, + store: true, + }), + conversationId: 'bound-conversation', + parentMessageId: 'bound-parent', + status: 'queued', + revision: 0, + }); + return storage.bindResponse(run.tenantId, fence(run), responseId, 102); +} + +it('renews the same fence idempotently without shortening a lease on reordered heartbeats', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + for (const now of [102, 102, 101]) + expect(await storage.renew(tenantId, fence(run), now, 1000)).toBe(true); + expect((await storage.read(tenantId, 'channel', run.runId))?.leaseExpiresAt).toBe(1102); + expect( + await storage.renew(tenantId, { ...fence(run), epoch: run.leaseEpoch + 1 }, 103, 1000), + ).toBe(false); + expect(await storage.renew(tenantId, fence(run), 1102, 1000)).toBe(false); + await expect(storage.renew(tenantId, fence(run), 103, -1)).rejects.toThrow( + 'Invalid channel lease', + ); + })); + +it.each(['teamId', 'appId', 'externalChannelId', 'externalThreadKey', 'agentId'] as const)( + 'isolates complete lane scope including %s in admission, lookup and cancellation', + (field) => + scope(async () => { + const first = event(); + const second = event('other'); + if (field === 'teamId' || field === 'appId') second.admission.actor[field] = 'different'; + else if (field === 'agentId') second.agentId = 'different'; + else second.admission[field] = 'different'; + const a = await storage.admit(first, 100); + const b = await storage.admit(second, 101); + expect(a.run.laneId).not.toBe(b.run.laneId); + const claimedA = (await storage.claim(tenantId, 'channel', 'a', 102, 1000))!; + const claimedB = (await storage.claim(tenantId, 'channel', 'b', 103, 1000))!; + expect((await storage.readActiveThread(first, 104))?.runId).toBe(claimedA.runId); + expect((await storage.readActiveThread(second, 104))?.runId).toBe(claimedB.runId); + expect(await storage.requestCancel(second, claimedA.runId)).toBe(false); + expect(await storage.requestCancel(first, claimedA.runId)).toBe(true); + const pa = await storage.prepareExecution(tenantId, fence(claimedA), 104); + const pb = await storage.prepareExecution(tenantId, fence(claimedB), 104); + expect(pa.execution?.conversationId).not.toBe(pb.execution?.conversationId); + }), +); + +it('rejects unbound and mismatched response projections and revalidates them before cursor commit', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + const snapshot = { + status: 'completed' as const, + text: 'answer', + elapsedMs: 1, + responseId: 'response', + }; + await expect(storage.saveSnapshot(tenantId, fence(run), snapshot, 103)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + await expect( + storage.saveSnapshot(tenantId, fence(run), { ...snapshot, responseId: undefined }, 103), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await bindTestResponse(run); + for (const status of [ + 'running', + 'requires_action', + 'completed', + 'failed', + 'cancelled', + ] as const) + await expect( + storage.saveSnapshot( + tenantId, + fence(run), + { ...snapshot, status, responseId: 'other' }, + 103, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.saveSnapshot(tenantId, fence(run), snapshot, 103); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'v1', + rendererVersion: 'test', + parts: [{ operationId: 'm', kind: 'message', required: true, text: 'answer' }], + }, + 104, + ); + await storage.beginDelivery(tenantId, fence(run), 'm', 105); + await storage.settleDelivery( + tenantId, + fence(run), + { operationId: 'm', attempts: 1, outcome: 'confirmed', externalMessageKey: 'C1:ok' }, + 106, + ); + await mongoose.models.ChannelRun.updateOne( + { runId: run.runId }, + { $set: { 'snapshot.responseId': 'poison' } }, + ); + await expect(storage.commitDelivery(tenantId, fence(run), 107)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + expect( + await mongoose.models.ChannelLane.findOne({ laneId: run.laneId }).lean(), + ).not.toHaveProperty('responseId'); + })); + +it.each(['message', 'artifact'] as const)( + 'requires %s evidence in settlement, recovery and persisted commit', + (kind) => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 10))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.saveSnapshot( + tenantId, + fence(run), + { + status: 'failed', + text: 'failure', + elapsedMs: 1, + artifacts: kind === 'artifact' ? [testArtifact] : [], + }, + 103, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'v1', + rendererVersion: 'test', + parts: [ + { + operationId: 'part', + kind, + required: true, + text: 'answer', + ...(kind === 'artifact' ? { artifactId: 'artifact' } : {}), + }, + ], + }, + 104, + ); + await storage.beginDelivery(tenantId, fence(run), 'part', 105); + if (kind === 'artifact') + await storage.checkpointUpload( + tenantId, + fence(run), + 'part', + 1, + { fileId: 'file', stage: 'allocated' }, + 105, + ); + const receipt = { operationId: 'part', attempts: 1, outcome: 'confirmed' as const }; + const wrong = + kind === 'artifact' ? { externalMessageKey: 'message' } : { externalFileId: 'file' }; + await expect( + storage.settleDelivery(tenantId, fence(run), { ...receipt, ...wrong }, 106), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + const replacement = (await storage.claim(tenantId, 'channel', 'replacement', 112, 1000))!; + await expect( + storage.reconcileDelivery(tenantId, fence(replacement), { ...receipt, ...wrong }, 113), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + if (kind === 'artifact') { + for (const evidence of [ + { externalFileId: 'other' }, + { externalFileId: 'file', upload: { fileId: 'other', stage: 'uploaded' as const } }, + ]) + await expect( + storage.reconcileDelivery( + tenantId, + fence(replacement), + { ...receipt, ...evidence }, + 113, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + } + const valid = + kind === 'artifact' ? { externalFileId: 'file' } : { externalMessageKey: 'message' }; + await storage.reconcileDelivery(tenantId, fence(replacement), { ...receipt, ...valid }, 113); + await storage.reconcileDelivery(tenantId, fence(replacement), { ...receipt, ...valid }, 114); + await mongoose.models.ChannelRun.updateOne( + { runId: run.runId }, + { $set: { receipts: [{ ...receipt, ...wrong }] } }, + ); + await expect(storage.commitDelivery(tenantId, fence(replacement), 115)).rejects.toMatchObject( + { code: 'invalid_transition' }, + ); + await mongoose.models.ChannelRun.updateOne( + { runId: run.runId }, + { $set: { receipts: [{ ...receipt, ...valid }] } }, + ); + expect((await storage.commitDelivery(tenantId, fence(replacement), 116)).state).toBe( + 'failed', + ); + }), +); + +it.each([undefined, null, ''])( + 'binds tenantless responses stored with tenantId %s', + (storedTenant) => + tenantStorage.run({}, async () => { + const input = { ...event(), tenantId: '_' }; + await storage.admit(input, 100); + const run = (await storage.claim('_', 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution('_', fence(run), 102); + await runAsSystem(async () => + mongoose.models.StoredResponse.collection.insertOne({ + ...(storedTenant === undefined ? {} : { tenantId: storedTenant }), + userId: 'actor', + agentId: 'agent', + responseId: 'response', + requestKey: `channel:${run.runId}`, + fingerprint: 'request', + requestJSON: JSON.stringify({ model: 'agent', input: 'hello', background: true }), + conversationId: 'canonical', + parentMessageId: 'parent', + status: 'queued', + revision: 0, + }), + ); + expect( + (await storage.bindResponse('_', fence(run), 'response', 103)).execution?.responseId, + ).toBe('response'); + }), +); + +it('injects the configured claim batch without starving a later eligible lane', () => + scope(async () => { + await storage.admit(event('first'), 100); + await storage.claim(tenantId, 'channel', 'active', 101, 1000); + for (let i = 0; i < 3; i++) await storage.admit(event(`blocked-${i}`), 102 + i); + const other = await storage.admit(event('ready', 'another-actor'), 106); + const small = createChannelRunStorage(mongoose, { claimBatchSize: 2 }); + expect(await small.claim(tenantId, 'channel', 'small', 107, 1000)).toBeNull(); + const large = createChannelRunStorage(mongoose, { claimBatchSize: 4 }); + expect((await large.claim(tenantId, 'channel', 'large', 108, 1000))?.runId).toBe( + other.run.runId, + ); + expect(() => createChannelRunStorage(mongoose, { claimBatchSize: 0 })).toThrow(); + })); + +const testArtifact = { + artifactId: 'artifact', + filename: 'report.txt', + mediaType: 'text/plain', + bytes: 12, + sha256: 'hash', +}; + +it('migrates the prototype event constraint before admitting colliding event IDs from different apps', () => + scope(async () => { + const collection = mongoose.models.ChannelRun.collection; + const legacy = { + tenantId: 1, + channelId: 1, + 'admission.actor.teamId': 1, + 'admission.eventId': 1, + }; + await collection.createIndex(legacy, { unique: true, name: 'prototype_event' }); + const revised = createChannelRunStorage(mongoose); + const second = event(); + second.admission.actor.appId = 'other-app'; + const [a, b] = await Promise.all([revised.admit(event(), 100), revised.admit(second, 100)]); + expect(a.run.runId).not.toBe(b.run.runId); + expect((await revised.admit(second, 101)).run.runId).toBe(b.run.runId); + const indexes = await collection.indexes(); + expect(indexes.some((index) => index.name === 'prototype_event')).toBe(false); + expect(indexes.find((index) => index.name === 'channel_run_event_v2')).toMatchObject({ + unique: true, + key: { 'admission.actor.appId': 1 }, + }); + })); + +it.each(['completed', 'failed', 'cancelled'] as const)( + 'never rolls a %s snapshot back into execution', + (status) => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await bindTestResponse(run); + for (const current of ['running', 'requires_action', 'running'] as const) + await storage.saveSnapshot( + tenantId, + fence(run), + { status: current, responseId: 'response', text: 'working', elapsedMs: 1 }, + 103, + ); + const terminal = { status, responseId: 'response', text: 'final', elapsedMs: 2 }; + await storage.saveSnapshot(tenantId, fence(run), terminal, 104); + for (const stale of ['running', 'requires_action'] as const) + await expect( + storage.saveSnapshot(tenantId, fence(run), { ...terminal, status: stale }, 105), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + expect( + (await storage.saveSnapshot(tenantId, fence(run), terminal, 106)).snapshot, + ).toMatchObject(terminal); + expect((await storage.read(tenantId, 'channel', run.runId))?.state).toBe('delivering'); + }), +); + +function testSummary(coverage: 'partial' | 'complete'): ChannelRunSummary { + return { + version: 1, + scope: 'actor_lane', + throughSequence: 1, + includesCurrent: true, + priorRuns: 0, + expectedRuns: 1, + observedRuns: 1, + costUSD: { coverage: 'unavailable', coveredRuns: 0 }, + tokens: { coverage: 'unavailable', coveredRuns: 0 }, + elapsedMs: { + coverage, + coveredRuns: 1, + ...(coverage === 'complete' ? { value: 2 } : { knownValue: 1 }), + }, + }; +} + +it('replaces partial summary metrics with supplied complete metrics and preserves them only on omission', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await bindTestResponse(run); + await storage.saveSnapshot( + tenantId, + fence(run), + { + status: 'running', + responseId: 'response', + text: 'working', + elapsedMs: 1, + summary: testSummary('partial'), + }, + 103, + ); + const terminal = { + status: 'completed' as const, + responseId: 'response', + text: 'answer', + elapsedMs: 2, + }; + const complete = await storage.saveSnapshot( + tenantId, + fence(run), + { ...terminal, summary: testSummary('complete') }, + 104, + ); + expect(complete.snapshot?.summary?.elapsedMs).toMatchObject({ coverage: 'complete', value: 2 }); + const replay = await storage.saveSnapshot(tenantId, fence(run), terminal, 105); + expect(replay.snapshot?.summary).toEqual(testSummary('complete')); + })); + +it('authorizes every artifact plan part against the execution snapshot before accepting transport work', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'failed', text: 'failure', elapsedMs: 1, artifacts: [testArtifact] }, + 103, + ); + const part = { operationId: 'file', kind: 'artifact' as const, required: true, text: 'report' }; + for (const artifactId of [undefined, 'other']) + await expect( + storage.savePlan( + tenantId, + fence(run), + { revision: 'p', rendererVersion: 'test', parts: [{ ...part, artifactId }] }, + 104, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.savePlan( + tenantId, + fence(run), + { revision: 'p', rendererVersion: 'test', parts: [{ ...part, artifactId: 'artifact' }] }, + 104, + ); + await mongoose.models.ChannelRun.updateOne( + { runId: run.runId }, + { $set: { 'plan.parts.0.artifactId': 'other' } }, + ); + await expect(storage.beginDelivery(tenantId, fence(run), 'file', 105)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + await expect( + storage.checkpointUpload( + tenantId, + fence(run), + 'file', + 0, + { fileId: 'file', stage: 'allocated' }, + 105, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('finalizes permanent required failure after active sends settle, releases the lane and preserves its old cursor', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.attachResponseCredential( + tenantId, + fence(run), + { apiKeyId: 'key', encryptedApiKey: 'private-ciphertext' }, + 102, + ); + await bindTestResponse(run); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'completed', responseId: 'response', text: 'answer', elapsedMs: 1 }, + 103, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'p', + rendererVersion: 'test', + parts: [ + { operationId: 'optional', kind: 'message', required: false, text: 'preview' }, + { operationId: 'required', kind: 'message', required: true, text: 'answer' }, + ], + }, + 104, + ); + await storage.beginDelivery(tenantId, fence(run), 'optional', 105); + await storage.beginDelivery(tenantId, fence(run), 'required', 105); + await storage.settleDelivery( + tenantId, + fence(run), + { operationId: 'required', attempts: 1, outcome: 'permanent_failure' }, + 106, + ); + await expect(storage.commitDelivery(tenantId, fence(run), 107)).rejects.toMatchObject({ + code: 'delivery_incomplete', + }); + await storage.settleDelivery( + tenantId, + fence(run), + { operationId: 'optional', attempts: 1, outcome: 'unknown' }, + 108, + ); + const failed = await storage.commitDelivery(tenantId, fence(run), 109); + expect(failed.state).toBe('failed'); + expect(failed.errorCode).toBe('delivery_permanent_failure'); + expect(failed.execution).not.toHaveProperty('encryptedApiKey'); + expect(failed).not.toHaveProperty('leaseOwner'); + await storage.admit(event('next'), 110); + const next = (await storage.claim(tenantId, 'channel', 'next', 111, 1000))!; + expect( + (await storage.prepareExecution(tenantId, fence(next), 112)).execution?.parentMessageId, + ).toBe('00000000-0000-0000-0000-000000000000'); + await storage.commitDelivery(tenantId, fence(run), 113); + expect(await mongoose.models.ChannelLane.findOne({ laneId: run.laneId }).lean()).toMatchObject({ + activeRunId: next.runId, + }); + })); + +it.each(['completed', 'failed', 'cancelled'] as const)( + 'retires credential ciphertext after delivering a %s execution', + (status) => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + const credential = { apiKeyId: 'key', encryptedApiKey: 'private-ciphertext' }; + await storage.attachResponseCredential(tenantId, fence(run), credential, 102); + await storage.attachResponseCredential(tenantId, fence(run), credential, 102); + await expect( + storage.attachResponseCredential( + tenantId, + fence(run), + { ...credential, apiKeyId: 'replacement' }, + 102, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await bindTestResponse(run); + await storage.saveSnapshot( + tenantId, + fence(run), + { status, responseId: 'response', text: 'answer', elapsedMs: 1 }, + 103, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'p', + rendererVersion: 'test', + parts: [{ operationId: 'm', kind: 'message', required: true, text: 'answer' }], + }, + 104, + ); + await storage.beginDelivery(tenantId, fence(run), 'm', 105); + await storage.settleDelivery( + tenantId, + fence(run), + { operationId: 'm', attempts: 1, outcome: 'confirmed', externalMessageKey: 'C1:reply' }, + 106, + ); + const result = await storage.commitDelivery(tenantId, fence(run), 107); + expect(result.execution).not.toHaveProperty('encryptedApiKey'); + expect((await storage.read(tenantId, 'channel', run.runId))?.execution).not.toHaveProperty( + 'encryptedApiKey', + ); + expect(await storage.findDeliveredReply(event(), 'C1:reply')).toEqual( + status === 'completed' ? result : null, + ); + const wrong = event(); + wrong.admission.actor.appId = 'other-app'; + expect(await storage.findDeliveredReply(wrong, 'C1:reply')).toBeNull(); + }), +); + +it('retires credentials when authority is lost before execution starts', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.attachResponseCredential( + tenantId, + fence(run), + { apiKeyId: 'key', encryptedApiKey: 'private-ciphertext' }, + 102, + ); + await storage.markAuthorityLost(tenantId, fence(run), 103); + await storage.finishAuthorityLost(tenantId, fence(run), 104); + const final = await storage.read(tenantId, 'channel', run.runId); + expect(final?.state).toBe('cancelled'); + expect(final?.execution).not.toHaveProperty('encryptedApiKey'); + })); + +it('distinguishes two apps whose delivered provider-local reply keys collide', () => + scope(async () => { + const first = event('one'); + const second = event('two'); + second.admission.actor.appId = 'another-app'; + const a = await storage.admit(first, 100); + const b = await storage.admit(second, 100); + for (const item of [a, b]) + await mongoose.models.ChannelRun.updateOne( + { runId: item.run.runId }, + { + $set: { + state: 'delivered', + receipts: [ + { + operationId: 'm', + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C1:same-key', + }, + ], + }, + }, + ); + expect((await storage.findDeliveredReply(first, 'C1:same-key'))?.runId).toBe(a.run.runId); + expect((await storage.findDeliveredReply(second, 'C1:same-key'))?.runId).toBe(b.run.runId); + })); + +it('drains a previously accepted plan even when the configured new-plan limit is lowered', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'failed', text: 'failure', elapsedMs: 1 }, + 103, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'p', + rendererVersion: 'test', + parts: [ + { operationId: 'm', kind: 'message', required: true, text: 'failure' }, + { operationId: 'o', kind: 'message', required: false, text: 'footer' }, + ], + }, + 104, + ); + const reduced = createChannelRunStorage(mongoose, { maxDeliveryParts: 1 }); + await reduced.beginDelivery(tenantId, fence(run), 'm', 105); + await reduced.settleDelivery( + tenantId, + fence(run), + { operationId: 'm', attempts: 1, outcome: 'confirmed', externalMessageKey: 'C1:reply' }, + 106, + ); + expect((await reduced.commitDelivery(tenantId, fence(run), 107)).state).toBe('failed'); + })); + +it('replays exact settled transport acknowledgements without opening another attempt', () => + scope(async () => { + const run = await plannedRun(); + await storage.beginDelivery(tenantId, fence(run), 'message', 105); + const receipt = { + operationId: 'message', + outcome: 'confirmed' as const, + attempts: 1, + externalMessageKey: 'C1:reply', + }; + await storage.settleDelivery(tenantId, fence(run), receipt, 106); + await storage.settleDelivery(tenantId, fence(run), receipt, 107); + await expect( + storage.settleDelivery( + tenantId, + fence(run), + { ...receipt, externalMessageKey: 'C1:other' }, + 108, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + const snapshot = (await storage.read(tenantId, 'channel', run.runId))!.snapshot!; + await storage.saveSnapshot(tenantId, fence(run), snapshot, 108); + await expect( + storage.saveSnapshot(tenantId, fence(run), { ...snapshot, text: 'changed' }, 108), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + expect((await storage.commitDelivery(tenantId, fence(run), 109)).state).toBe('delivered'); + })); + +it('refuses every named underscore context before it can read or mutate tenantless runs', async () => { + const input = { ...event(), tenantId: '_' }; + const { run } = await tenantStorage.run({}, async () => storage.admit(input, 100)); + for (const action of [ + () => storage.read('_', 'channel', run.runId), + () => storage.claim('_', 'channel', 'worker', 101, 1000), + () => storage.listPendingMetadata(undefined, 10), + () => storage.requestCancel(input, run.runId), + () => storage.readActiveThread(input, 101), + () => storage.findDeliveredReply(input, 'C1:reply'), + ]) + await expect(tenantStorage.run({ tenantId: '_' }, async () => action())).rejects.toThrow( + 'Reserved tenant identifier', + ); + expect( + await tenantStorage.run({}, async () => storage.read('_', 'channel', run.runId)), + ).toMatchObject({ state: 'queued', leaseEpoch: 0, cancellationRequested: false }); +}); + +it('rejects persisting a real user in the reserved channel partition', () => + runAsSystem(async () => { + await expect( + mongoose.models.User.create({ + email: 'reserved-tenant@example.com', + provider: 'local', + tenantId: '_', + }), + ).rejects.toThrow('Reserved tenant identifier'); + })); + +it.each(['principal', 'agent'] as const)( + 'normalizes concurrent %s conflicts on one provider event', + (kind) => + scope(async () => { + const first = event(); + const second = event(); + if (kind === 'principal') second.principal.userId = 'different-owner'; + else second.agentId = 'different-agent'; + const results = await Promise.allSettled([ + storage.admit(first, 100), + storage.admit(second, 100), + ]); + expect(results.filter((result) => result.status === 'fulfilled')).toHaveLength(1); + const failure = results.find((result) => result.status === 'rejected'); + expect(failure).toMatchObject({ + status: 'rejected', + reason: { name: 'ChannelRunConflict', code: 'event_conflict' }, + }); + expect(await mongoose.models.ChannelRun.countDocuments({ tenantId })).toBe(1); + expect(await mongoose.models.ChannelLane.countDocuments({ tenantId })).toBe(1); + }), +); + +it('injects account erasure work bounds before closing admission', () => + scope(async () => { + const user = await mongoose.models.User.create({ + email: 'configured-erasure@example.com', + provider: 'local', + tenantId, + }); + const userId = String(user._id); + for (let i = 0; i < 3; i++) { + const input = event(`erase-${i}`); + input.principal.userId = userId; + input.admission.externalThreadKey = `thread-${i}`; + await storage.admit(input, 100 + i); + } + await mongoose.models.ChannelLane.updateMany({ userId }, { $unset: { userId: 1 } }); + const getConfig = jest.fn(async () => ({ erasureBatchSize: 1 })); + const deleted = jest.spyOn(mongoose.models.ChannelLane, 'deleteMany'); + try { + await createUserMethods(mongoose, { getChannelStorageConfig: getConfig }).deleteUserById( + userId, + ); + const batches = deleted.mock.calls + .map(([filter]) => filter as { laneId?: { $in?: string[] } }) + .filter((filter) => filter?.laneId?.$in); + expect(batches).toHaveLength(3); + for (const batch of batches) expect(batch?.laneId?.$in).toHaveLength(1); + expect(getConfig).toHaveBeenCalledTimes(1); + } finally { + deleted.mockRestore(); + } + })); + +it('fails a bad erasure configuration before closing the account owner fence', () => + scope(async () => { + const user = await mongoose.models.User.create({ + email: 'bad-erasure-config@example.com', + provider: 'local', + tenantId, + }); + const userId = String(user._id); + await expect( + createUserMethods(mongoose, { + getChannelStorageConfig: async () => ({ erasureBatchSize: 0 }), + }).deleteUserById(userId), + ).rejects.toThrow(); + const input = event(); + input.principal.userId = userId; + await expect(storage.admit(input, 100)).resolves.toMatchObject({ duplicate: false }); + })); + +it('persists cancellation as language-neutral status without presentation copy', () => + scope(async () => { + await storage.admit(event(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 101, 1000))!; + await storage.prepareExecution(tenantId, fence(run), 102); + await storage.requestCancel(event(), run.runId); + const result = await storage.finishUnstartedResponseCancellation(tenantId, fence(run), 103); + expect(result.snapshot).toMatchObject({ status: 'cancelled', text: '' }); + })); diff --git a/packages/data-schemas/src/methods/channelRun.ts b/packages/data-schemas/src/methods/channelRun.ts new file mode 100644 index 0000000000..5b04092359 --- /dev/null +++ b/packages/data-schemas/src/methods/channelRun.ts @@ -0,0 +1,1072 @@ +import { randomUUID } from 'node:crypto'; +import { + channelStorageConfigSchema, + CHANNEL_DELIVERY_PARTS_HARD_MAX, +} from 'librechat-data-provider'; +import type { + ChannelStorageConfig, + ChannelRunFence, + ChannelRunSnapshot, + ChannelDeliveryPlan, + ChannelDeliveryReceipt, + ChannelRunProgress, +} from 'librechat-data-provider'; +import type { ClientSession } from 'mongoose'; +import type { ChannelLaneScope, ChannelRunEnrollment } from './channelProtocol'; +import type { ChannelRunRecord, ChannelLaneRecord } from '~/models/channelRun'; +import type { StoredResponseRecord } from '~/models/response'; +import { + digest, + laneKey, + activeStates, + assertReconciliation, + assertDeliveryEvidence, + snapshotPatch, + assertPlan, + deliveryDisposition, + hasPermanentDeliveryFailure, + leaseDeadline, + ChannelRunConflict, +} from './channelProtocol'; +import { prepareChannelOwnerAdmission } from './channelOwner'; +import { createChannelIndexGate } from './channelIndexes'; +import { responseOwnerFilter } from './responseScope'; +import { getTenantId } from '~/config/tenantContext'; + +export type { ChannelRunRecord, ChannelLaneRecord } from '~/models/channelRun'; + +export type { ChannelLaneScope, ChannelRunEnrollment } from './channelProtocol'; +export { ChannelRunConflict } from './channelProtocol'; + +export interface ChannelRunStorage { + listPendingMetadata( + cursor: string | undefined, + limit: number, + ): Promise<{ + runs: Array<{ tenantId: string; channelId: string; runId: string }>; + next?: string; + }>; + claimAuthorityLost( + run: ChannelRunRecord, + owner: string, + now: number, + leaseMs: number, + ): Promise; + admit( + input: ChannelRunEnrollment, + now: number, + ): Promise<{ run: ChannelRunRecord; duplicate: boolean }>; + claim( + tenantId: string, + channelId: string, + owner: string, + now: number, + leaseMs: number, + ): Promise; + renew(tenantId: string, fence: ChannelRunFence, now: number, leaseMs: number): Promise; + /** Yield only a drained projection turn, never its execution or actor lane. */ + yieldLease( + tenantId: string, + channelId: string, + fence: ChannelRunFence, + now: number, + ): Promise; + read(tenantId: string, channelId: string, runId: string): Promise; + readActiveThread(scope: ChannelLaneScope, before: number): Promise; + prepareExecution( + tenantId: string, + fence: ChannelRunFence, + now: number, + ): Promise; + bindResponse( + tenantId: string, + fence: ChannelRunFence, + responseId: string, + now: number, + ): Promise; + /** Stop-only reconciliation. Absence is not proof that response creation has retired. */ + recoverResponse( + tenantId: string, + fence: ChannelRunFence, + now: number, + ): Promise; + attachResponseCredential( + tenantId: string, + fence: ChannelRunFence, + credential: { apiKeyId: string; encryptedApiKey: string }, + now: number, + ): Promise; + finishUnstartedResponseCancellation( + tenantId: string, + fence: ChannelRunFence, + now: number, + ): Promise; + saveSnapshot( + tenantId: string, + fence: ChannelRunFence, + snapshot: ChannelRunSnapshot, + now: number, + generationCreatedAt?: number, + ): Promise; + savePlan( + tenantId: string, + fence: ChannelRunFence, + plan: ChannelDeliveryPlan, + now: number, + ): Promise; + planProgress( + tenantId: string, + fence: ChannelRunFence, + plan: Pick, + now: number, + ): Promise; + beginProgress( + tenantId: string, + fence: ChannelRunFence, + revision: number, + now: number, + ): Promise; + settleProgress( + tenantId: string, + fence: ChannelRunFence, + receipt: Omit, + now: number, + ): Promise; + reconcileProgress( + tenantId: string, + fence: ChannelRunFence, + receipt: Pick, + now: number, + ): Promise; + beginDelivery( + tenantId: string, + fence: ChannelRunFence, + operationId: string, + now: number, + ): Promise; + settleDelivery( + tenantId: string, + fence: ChannelRunFence, + receipt: ChannelDeliveryReceipt, + now: number, + ): Promise; + /** Confirm a previously uncertain send only after verifying its provider receipt. Never resends. */ + reconcileDelivery( + tenantId: string, + fence: ChannelRunFence, + receipt: ChannelDeliveryReceipt & { outcome: 'confirmed' }, + now: number, + ): Promise; + checkpointUpload( + tenantId: string, + fence: ChannelRunFence, + operationId: string, + attempts: number, + upload: NonNullable, + now: number, + ): Promise; + commitDelivery(tenantId: string, fence: ChannelRunFence, now: number): Promise; + markAuthorityLost( + tenantId: string, + fence: ChannelRunFence, + now: number, + ): Promise; + finishAuthorityLost(tenantId: string, fence: ChannelRunFence, now: number): Promise; + requestCancel( + scope: ChannelLaneScope, + runId: string, + observed?: { epoch: number; before: number }, + ): Promise; + findDeliveredReply( + scope: ChannelLaneScope, + externalMessageKey: string, + ): Promise; +} + +function authorityLost(run: ChannelRunRecord): Partial { + return { + cancellationRequested: true, + errorCode: 'channel_authority_changed', + receipts: run.receipts.map((receipt) => + receipt.outcome === 'inflight' ? { ...receipt, outcome: 'unknown' as const } : receipt, + ), + ...(run.progress?.outcome === 'inflight' + ? { progress: { ...run.progress, outcome: 'unknown' as const } } + : {}), + }; +} + +/** Mongo transactions protect FIFO and cursor/receipt commit; a standalone Mongo fails closed. */ +export function createChannelRunStorage( + mongoose: typeof import('mongoose'), + options: Partial = {}, +): ChannelRunStorage { + const limits = channelStorageConfigSchema.parse(options); + const runs = () => mongoose.model('ChannelRun'); + const lanes = () => mongoose.model('ChannelLane'); + const ensureIndexes = createChannelIndexGate( + () => [runs(), lanes()], + async () => { + // The prototype index was narrower than provider event identity. Build its + // replacement first, then remove only that exact old unique constraint. + const legacy = { + tenantId: 1, + channelId: 1, + 'admission.actor.teamId': 1, + 'admission.eventId': 1, + }; + for (const index of await runs().listIndexes()) { + if (!index.unique || !index.name || digest(index.key) !== digest(legacy)) continue; + try { + await runs().cleanIndexes({ toDrop: [index.name] }); + } catch (error) { + if (!(error instanceof mongoose.mongo.MongoServerError) || error.code !== 27) throw error; + } + } + }, + ); + async function transaction(work: (session: ClientSession) => Promise): Promise { + await ensureIndexes(); + const session = await mongoose.startSession(); + try { + let result: T | undefined; + await session.withTransaction(async () => { + result = await work(session); + }); + if (result === undefined) throw new Error('Channel transaction did not commit'); + return result; + } finally { + await session.endSession(); + } + } + const terminalUnset = { leaseOwner: 1, 'execution.encryptedApiKey': 1 } as const; + const predicate = (tenantId: string, fence: ChannelRunFence, now: number) => ({ + tenantId, + runId: fence.runId, + leaseOwner: fence.owner, + leaseEpoch: fence.epoch, + leaseExpiresAt: { $gt: now }, + state: { $in: activeStates }, + }); + async function owned( + tenantId: string, + fence: ChannelRunFence, + now: number, + session: ClientSession, + ) { + const run = await runs() + .findOne(predicate(tenantId, fence, now)) + .session(session) + .lean(); + if (!run) throw new ChannelRunConflict('stale_fence'); + return run; + } + async function mutate( + tenantId: string, + fence: ChannelRunFence, + now: number, + change: (run: ChannelRunRecord, session: ClientSession) => Promise>, + ): Promise { + return transaction(async (session) => { + const run = await owned(tenantId, fence, now, session); + const patch = await change(run, session); + const result = await runs() + .findOneAndUpdate(predicate(tenantId, fence, now), { $set: patch }, { new: true, session }) + .lean(); + if (!result) throw new ChannelRunConflict('stale_fence'); + return result; + }); + } + async function responseBinding( + run: ChannelRunRecord, + session: ClientSession, + responseId?: string, + ): Promise | null> { + if ( + !run.execution || + (run.state !== 'executing' && run.execution.protocol !== 'responses') || + run.execution.deliveryKey != null || + run.execution.enqueueAttemptedAt != null || + (run.execution.responseId != null && run.execution.responseId !== responseId) + ) + throw new ChannelRunConflict('invalid_transition'); + const response = await mongoose + .model('StoredResponse') + .findOne({ + ...responseOwnerFilter({ + tenantId: run.tenantId === '_' ? undefined : run.tenantId, + userId: run.principal.userId, + }), + agentId: run.agentId, + ...(responseId === undefined ? {} : { responseId }), + requestKey: `channel:${run.runId}`, + ...(run.execution.apiKeyId ? { sourceKeyId: run.execution.apiKeyId } : {}), + }) + .session(session) + .lean(); + if (!response) return null; + const request: { + input?: string; + model?: string; + previous_response_id?: string; + background?: boolean; + store?: boolean; + } = JSON.parse(response.requestJSON); + const previous = + run.execution.parentMessageId === '00000000-0000-0000-0000-000000000000' + ? undefined + : run.execution.parentMessageId; + if ( + request.model !== run.agentId || + request.input !== run.admission.input || + (run.execution.responseId == null && request.previous_response_id !== previous) || + request.background !== true || + request.store === false + ) + throw new ChannelRunConflict('invalid_transition'); + return { + execution: { + ...run.execution, + protocol: 'responses', + responseId: response.responseId, + conversationId: response.conversationId, + parentMessageId: response.parentMessageId, + ...(response.generationCreatedAt == null + ? {} + : { generationCreatedAt: response.generationCreatedAt }), + }, + }; + } + return { + async listPendingMetadata(cursor, limit) { + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + limit > limits.maxMetadataPageSize || + (cursor && !mongoose.isValidObjectId(cursor)) + ) + throw new Error('Invalid run metadata page'); + await ensureIndexes(); + const records = await runs() + .find({ state: { $in: activeStates }, ...(cursor ? { _id: { $gt: cursor } } : {}) }) + .select({ _id: 1, tenantId: 1, channelId: 1, runId: 1 }) + .sort({ _id: 1 }) + .limit(limit + 1) + .lean(); + const page = records.slice(0, limit); + return { + runs: page.map(({ tenantId, channelId, runId }) => ({ tenantId, channelId, runId })), + ...(records.length > limit ? { next: String(page[page.length - 1]._id) } : {}), + }; + }, + async claimAuthorityLost(run, owner, now, leaseMs) { + if (!owner || !Number.isSafeInteger(leaseMs) || leaseMs <= 0) + throw new Error('Invalid cleanup lease'); + return transaction(async (session) => { + const filter = { + tenantId: run.tenantId, + channelId: run.channelId, + runId: run.runId, + leaseEpoch: run.leaseEpoch, + state: { $in: activeStates }, + $or: [ + { errorCode: { $ne: 'channel_authority_changed' } }, + { leaseExpiresAt: { $lte: now } }, + ], + }; + const current = await runs().findOne(filter).session(session).lean(); + if (!current) return null; + return runs() + .findOneAndUpdate( + filter, + { + $set: { + ...authorityLost(current), + leaseOwner: owner, + leaseExpiresAt: leaseDeadline(now, leaseMs), + }, + $inc: { leaseEpoch: 1 }, + }, + { new: true, session }, + ) + .lean(); + }); + }, + async admit(input, now) { + const ownerAdmission = await prepareChannelOwnerAdmission(mongoose, { + userId: input.principal.userId, + tenantId: input.tenantId === '_' ? undefined : input.tenantId, + }); + const eventKey = { + tenantId: input.tenantId, + channelId: input.channelId, + 'admission.actor.teamId': input.admission.actor.teamId, + 'admission.actor.appId': input.admission.actor.appId, + 'admission.eventId': input.admission.eventId, + }; + const payloadHash = digest(input.admission); + const laneId = laneKey(input); + try { + return await transaction(async (session) => { + if (!(await ownerAdmission(session))) throw new ChannelRunConflict('invalid_transition'); + const existing = await runs().findOne(eventKey).session(session).lean(); + if (existing) { + if ( + digest(existing.admission) !== payloadHash || + existing.agentId !== input.agentId || + digest(existing.principal) !== digest(input.principal) + ) + throw new ChannelRunConflict('event_conflict'); + return { run: existing, duplicate: true }; + } + const lane = await lanes() + .findOneAndUpdate( + { tenantId: input.tenantId, laneId }, + { + $inc: { nextSequence: 1 }, + $setOnInsert: { + channelId: input.channelId, + userId: input.principal.userId, + }, + }, + { upsert: true, new: true, session }, + ) + .lean(); + if (!lane) throw new Error('Channel lane unavailable'); + const [record] = await runs().create( + [ + { + ...input, + runId: randomUUID(), + laneId, + payloadHash, + sequence: lane.nextSequence, + receivedAt: now, + state: 'queued', + leaseEpoch: 0, + leaseExpiresAt: 0, + receipts: [], + cancellationRequested: false, + }, + ], + { session }, + ); + return { run: record.toObject(), duplicate: false }; + }); + } catch (error) { + if ( + !(error instanceof mongoose.mongo.MongoServerError) || + error.code !== 11000 || + (error.keyPattern?.['admission.eventId'] == null && + !error.message.includes('channel_run_event_v2')) + ) + throw error; + const existing = await runs().findOne(eventKey).lean(); + if (!existing) throw new ChannelRunConflict('event_conflict'); + if ( + digest(existing.admission) !== payloadHash || + existing.agentId !== input.agentId || + digest(existing.principal) !== digest(input.principal) + ) + throw new ChannelRunConflict('event_conflict'); + return { run: existing, duplicate: true }; + } + }, + async claim(tenantId, channelId, owner, now, leaseMs) { + if (!owner || !Number.isSafeInteger(leaseMs) || leaseMs <= 0) + throw new Error('Invalid channel lease'); + return transaction(async (session) => { + const candidates = await runs() + .find({ + tenantId, + channelId, + state: { $in: activeStates }, + leaseExpiresAt: { $lte: now }, + }) + .sort({ claimCheckedAt: 1, receivedAt: 1, sequence: 1 }) + .hint('channel_run_fair_claim') + .limit(limits.claimBatchSize) + .session(session) + .lean(); + const laneCache = new Map(); + for (const run of candidates) { + if (!laneCache.has(run.laneId)) + laneCache.set( + run.laneId, + await lanes().findOne({ tenantId, laneId: run.laneId }).session(session).lean(), + ); + const lane = laneCache.get(run.laneId); + if ( + !lane || + (lane.activeRunId && lane.activeRunId !== run.runId) || + (await runs() + .exists({ + tenantId, + laneId: run.laneId, + sequence: { $lt: run.sequence }, + state: { $in: activeStates }, + }) + .hint('channel_run_active_predecessor') + .session(session)) + ) { + await runs().updateOne( + { + tenantId, + runId: run.runId, + leaseEpoch: run.leaseEpoch, + leaseExpiresAt: run.leaseExpiresAt, + }, + { $set: { claimCheckedAt: now } }, + { session }, + ); + continue; + } + await lanes().updateOne( + { tenantId, laneId: run.laneId }, + { $set: { activeRunId: run.runId } }, + { session }, + ); + // An expired sender may have reached Slack. Recovery must never turn this into a repost. + const receipts = run.receipts.map((receipt) => + receipt.outcome === 'inflight' ? { ...receipt, outcome: 'unknown' as const } : receipt, + ); + return runs() + .findOneAndUpdate( + { tenantId, runId: run.runId, leaseEpoch: run.leaseEpoch }, + { + $set: { + leaseOwner: owner, + leaseExpiresAt: leaseDeadline(now, leaseMs), + claimCheckedAt: now, + receipts, + ...(run.progress?.outcome === 'inflight' + ? { progress: { ...run.progress, outcome: 'unknown' } } + : {}), + }, + $inc: { leaseEpoch: 1 }, + }, + { new: true, session }, + ) + .lean(); + } + return null; + }); + }, + async renew(tenantId, fence, now, leaseMs) { + const result = await runs().updateOne(predicate(tenantId, fence, now), { + $max: { leaseExpiresAt: leaseDeadline(now, leaseMs) }, + }); + return result.matchedCount === 1; + }, + async yieldLease(tenantId, channelId, fence, now) { + if ((getTenantId() ?? '_') !== tenantId) return false; + const result = await runs().updateOne( + { + ...predicate(tenantId, fence, now), + channelId, + state: { $in: ['executing', 'requires_action', 'blocked'] }, + 'execution.protocol': 'responses', + 'execution.responseId': { $type: 'string' }, + plan: { $exists: false }, + 'progress.outcome': { $nin: ['inflight', 'unknown'] }, + receipts: { $not: { $elemMatch: { outcome: { $in: ['inflight', 'unknown'] } } } }, + }, + { + // Clearing owner invalidates its fence immediately; expiry records the + // configured next eligibility, not continued execution ownership. + $set: { leaseExpiresAt: now + limits.projectionYieldMs }, + $unset: { leaseOwner: 1 }, + }, + ); + return result.modifiedCount === 1; + }, + async read(tenantId, channelId, runId) { + return runs().findOne({ tenantId, channelId, runId }).lean(); + }, + async prepareExecution(tenantId, fence, now) { + return mutate(tenantId, fence, now, async (run, session) => { + if (run.execution) return {}; + if (run.state !== 'queued') throw new ChannelRunConflict('invalid_transition'); + const lane = await lanes() + .findOne({ tenantId, laneId: run.laneId, activeRunId: run.runId }) + .session(session) + .lean(); + if (!lane) throw new ChannelRunConflict('stale_fence'); + return { + state: 'executing', + execution: { + requestId: run.runId, + conversationId: lane.conversationId ?? randomUUID(), + parentMessageId: lane.responseId ?? '00000000-0000-0000-0000-000000000000', + }, + }; + }); + }, + async finishUnstartedResponseCancellation(tenantId, fence, now) { + return mutate(tenantId, fence, now, async (run) => { + const execution = run.execution; + if ( + !run.cancellationRequested || + !execution || + execution.apiKeyId || + execution.responseId || + execution.generationCreatedAt != null || + execution.deliveryKey || + execution.enqueueAttemptedAt || + run.plan + ) + throw new ChannelRunConflict('invalid_transition'); + return { + state: 'delivering' as const, + execution: { ...execution, protocol: 'responses' as const }, + snapshot: { + status: 'cancelled' as const, + text: '', + elapsedMs: Math.max(0, now - run.receivedAt), + }, + }; + }); + }, + async attachResponseCredential(tenantId, fence, credential, now) { + return mutate(tenantId, fence, now, async (run) => { + if ( + !run.execution || + !['queued', 'executing', 'requires_action'].includes(run.state) || + run.execution.deliveryKey || + run.execution.enqueueAttemptedAt || + !credential.apiKeyId || + !credential.encryptedApiKey || + run.cancellationRequested + ) + throw new ChannelRunConflict('invalid_transition'); + if (run.execution.apiKeyId) { + if ( + run.execution.apiKeyId !== credential.apiKeyId || + run.execution.encryptedApiKey !== credential.encryptedApiKey + ) + throw new ChannelRunConflict('invalid_transition'); + return {}; + } + return { execution: { ...run.execution, protocol: 'responses' as const, ...credential } }; + }); + }, + async bindResponse(tenantId, fence, responseId, now) { + return mutate(tenantId, fence, now, async (run, session) => { + const patch = await responseBinding(run, session, responseId); + if (!patch) throw new ChannelRunConflict('invalid_transition'); + return patch; + }); + }, + async recoverResponse(tenantId, fence, now) { + if ((getTenantId() ?? '_') !== tenantId) throw new ChannelRunConflict('invalid_transition'); + return transaction(async (session) => { + const run = await owned(tenantId, fence, now, session); + if ( + !run.cancellationRequested || + run.execution?.protocol !== 'responses' || + !run.execution.apiKeyId || + run.execution.deliveryKey != null || + run.execution.enqueueAttemptedAt != null + ) + throw new ChannelRunConflict('invalid_transition'); + if (run.execution.responseId) return run; + const patch = await responseBinding(run, session); + if (!patch) return null; + const recovered = await runs() + .findOneAndUpdate( + { ...predicate(tenantId, fence, now), cancellationRequested: true }, + { $set: patch }, + { new: true, session }, + ) + .lean(); + if (!recovered) throw new ChannelRunConflict('stale_fence'); + return recovered; + }); + }, + async saveSnapshot(tenantId, fence, snapshot, now, generationCreatedAt) { + return mutate(tenantId, fence, now, async (run) => { + if (!run.execution) throw new ChannelRunConflict('invalid_transition'); + if ( + generationCreatedAt != null && + run.execution.generationCreatedAt != null && + generationCreatedAt !== run.execution.generationCreatedAt + ) + throw new ChannelRunConflict('invalid_transition'); + return { + ...snapshotPatch(run, snapshot), + execution: { + ...run.execution, + ...(generationCreatedAt != null ? { generationCreatedAt } : {}), + }, + }; + }); + }, + async savePlan(tenantId, fence, plan, now) { + return mutate(tenantId, fence, now, async (run) => { + if (run.plan) { + if (digest(run.plan) !== digest(plan)) throw new ChannelRunConflict('invalid_transition'); + return {}; + } + assertPlan(run, plan, limits.maxDeliveryParts); + return { + plan, + receipts: plan.parts.map(({ operationId }) => ({ + operationId, + outcome: 'planned', + attempts: 0, + })), + }; + }); + }, + async planProgress(tenantId, fence, plan, now) { + return mutate(tenantId, fence, now, async (run) => { + const previous = run.progress; + if (previous?.revision === plan.revision) { + if (previous.text !== plan.text || previous.rendererVersion !== plan.rendererVersion) + throw new ChannelRunConflict('invalid_transition'); + return {}; + } + const writable = + ['queued', 'executing', 'requires_action'].includes(run.state) || + (run.state === 'delivering' && Boolean(previous?.externalMessageKey)); + if ( + run.plan || + !writable || + plan.revision !== (previous?.revision ?? 0) + 1 || + (previous && previous.outcome !== 'confirmed') + ) { + throw new ChannelRunConflict('invalid_transition'); + } + return { + progress: { + ...plan, + outcome: 'planned', + attempts: 0, + ...(previous?.externalMessageKey + ? { externalMessageKey: previous.externalMessageKey } + : {}), + }, + }; + }); + }, + async beginProgress(tenantId, fence, revision, now) { + return mutate(tenantId, fence, now, async (run) => { + const progress = run.progress; + const writable = + ['queued', 'executing', 'requires_action'].includes(run.state) || + (run.state === 'delivering' && Boolean(progress?.externalMessageKey)); + if ( + run.plan || + !writable || + !progress || + progress.revision !== revision || + !['planned', 'retryable'].includes(progress.outcome) || + (progress.retryAt ?? 0) > now + ) + throw new ChannelRunConflict('invalid_transition'); + return { progress: { ...progress, outcome: 'inflight', attempts: progress.attempts + 1 } }; + }); + }, + async settleProgress(tenantId, fence, receipt, now) { + return mutate(tenantId, fence, now, async (run) => { + const previous = run.progress; + if ( + previous && + previous.outcome !== 'inflight' && + ['confirmed', 'retryable', 'unknown', 'permanent_failure'].includes(receipt.outcome) && + previous.attempts === receipt.attempts && + previous.outcome === receipt.outcome && + digest({ ...previous, ...receipt }) === digest(previous) + ) + return {}; + if ( + !previous || + previous.outcome !== 'inflight' || + previous.revision !== receipt.revision || + previous.attempts !== receipt.attempts || + !['confirmed', 'retryable', 'unknown', 'permanent_failure'].includes(receipt.outcome) || + (receipt.outcome === 'confirmed' && !receipt.externalMessageKey) || + (previous.externalMessageKey && + receipt.externalMessageKey && + previous.externalMessageKey !== receipt.externalMessageKey) + ) { + throw new ChannelRunConflict('invalid_transition'); + } + return { progress: { ...previous, ...receipt } }; + }); + }, + async beginDelivery(tenantId, fence, operationId, now) { + return mutate(tenantId, fence, now, async (run) => { + if (!run.plan) throw new ChannelRunConflict('invalid_transition'); + assertPlan(run, run.plan, CHANNEL_DELIVERY_PARTS_HARD_MAX); + if (hasPermanentDeliveryFailure(run)) throw new ChannelRunConflict('invalid_transition'); + const index = run.receipts.findIndex((receipt) => receipt.operationId === operationId); + const receipt = run.receipts[index]; + if ( + !run.plan || + !receipt || + !['planned', 'retryable'].includes(receipt.outcome) || + (receipt.retryAt ?? 0) > now + ) + throw new ChannelRunConflict('invalid_transition'); + if ( + run.receipts + .slice(0, index) + .some((prior, i) => run.plan?.parts[i].required && prior.outcome !== 'confirmed') + ) + throw new ChannelRunConflict('delivery_incomplete'); + return { + receipts: run.receipts.map((part) => + part.operationId === operationId + ? { ...part, outcome: 'inflight' as const, attempts: part.attempts + 1 } + : part, + ), + }; + }); + }, + async settleDelivery(tenantId, fence, receipt, now) { + return mutate(tenantId, fence, now, async (run) => { + const previous = run.receipts.find((part) => part.operationId === receipt.operationId); + if ( + previous && + previous.outcome !== 'inflight' && + ['confirmed', 'retryable', 'unknown', 'permanent_failure'].includes(receipt.outcome) && + previous.attempts === receipt.attempts && + previous.outcome === receipt.outcome && + digest({ ...previous, ...receipt }) === digest(previous) + ) + return {}; + if ( + !previous || + previous.outcome !== 'inflight' || + previous.attempts !== receipt.attempts || + !['confirmed', 'retryable', 'unknown', 'permanent_failure'].includes(receipt.outcome) || + (previous?.upload && + receipt.externalFileId && + previous.upload.fileId !== receipt.externalFileId) + ) + throw new ChannelRunConflict('invalid_transition'); + assertDeliveryEvidence(run, receipt, previous); + return { + receipts: run.receipts.map((part) => + part.operationId === receipt.operationId ? { ...part, ...receipt } : part, + ), + }; + }); + }, + async reconcileProgress(tenantId, fence, receipt, now) { + return mutate(tenantId, fence, now, async (run) => { + if (run.progress?.revision !== receipt.revision || !receipt.externalMessageKey) + throw new ChannelRunConflict('invalid_transition'); + assertReconciliation(run.progress, receipt); + return { + progress: { + ...run.progress, + outcome: 'confirmed' as const, + externalMessageKey: receipt.externalMessageKey, + retryAt: undefined, + errorCode: undefined, + }, + }; + }); + }, + async reconcileDelivery(tenantId, fence, receipt, now) { + return mutate(tenantId, fence, now, async (run) => { + const previous = run.receipts.find((part) => part.operationId === receipt.operationId); + if ( + !run.plan?.parts.some((part) => part.operationId === receipt.operationId) || + receipt.outcome !== 'confirmed' + ) + throw new ChannelRunConflict('invalid_transition'); + assertReconciliation(previous, receipt); + assertDeliveryEvidence(run, receipt, previous); + return { + receipts: run.receipts.map((part) => + part.operationId === receipt.operationId + ? { + ...part, + outcome: 'confirmed' as const, + externalMessageKey: receipt.externalMessageKey, + externalFileId: receipt.externalFileId, + retryAt: undefined, + errorCode: undefined, + } + : part, + ), + }; + }); + }, + async checkpointUpload(tenantId, fence, operationId, attempts, upload, now) { + return mutate(tenantId, fence, now, async (run) => { + if (!run.plan) throw new ChannelRunConflict('invalid_transition'); + assertPlan(run, run.plan, CHANNEL_DELIVERY_PARTS_HARD_MAX); + const previous = run.receipts.find((part) => part.operationId === operationId); + const part = run.plan?.parts.find((item) => item.operationId === operationId); + if ( + !previous || + previous.outcome !== 'inflight' || + previous.attempts !== attempts || + part?.kind !== 'artifact' || + !upload.fileId || + !['allocated', 'uploaded'].includes(upload.stage) || + (previous.externalFileId != null && previous.externalFileId !== upload.fileId) || + (previous.upload && + (previous.upload.fileId !== upload.fileId || + (previous.upload.stage === 'uploaded' && upload.stage !== 'uploaded'))) || + (!previous.upload && upload.stage !== 'allocated') + ) + throw new ChannelRunConflict('invalid_transition'); + return { + receipts: run.receipts.map((item) => + item.operationId === operationId ? { ...item, upload } : item, + ), + }; + }); + }, + async commitDelivery(tenantId, fence, now) { + return transaction(async (session) => { + const existing = await runs() + .findOne({ + tenantId, + runId: fence.runId, + state: { $in: ['delivered', 'failed', 'cancelled'] }, + }) + .session(session) + .lean(); + if (existing) { + await runs().updateOne( + { tenantId, runId: existing.runId, state: existing.state }, + { $unset: terminalUnset }, + { session }, + ); + if (existing.execution) delete existing.execution.encryptedApiKey; + delete existing.leaseOwner; + return existing; + } + const run = await owned(tenantId, fence, now, session); + const state = deliveryDisposition(run, CHANNEL_DELIVERY_PARTS_HARD_MAX); + const success = state === 'delivered'; + const laneUpdate = await lanes().updateOne( + { tenantId, laneId: run.laneId, activeRunId: run.runId }, + { + $unset: { activeRunId: 1 }, + ...(success + ? { + $set: { + conversationId: run.execution?.conversationId, + responseId: run.execution?.responseId, + }, + } + : {}), + }, + { session }, + ); + if (laneUpdate.modifiedCount !== 1) throw new ChannelRunConflict('stale_fence'); + const result = await runs() + .findOneAndUpdate( + predicate(tenantId, fence, now), + { + $set: { + state, + ...(hasPermanentDeliveryFailure(run) + ? { errorCode: 'delivery_permanent_failure' } + : {}), + deliveredAt: now, + leaseExpiresAt: 0, + }, + $unset: terminalUnset, + }, + { new: true, session }, + ) + .lean(); + if (!result) throw new ChannelRunConflict('stale_fence'); + return result; + }); + }, + async markAuthorityLost(tenantId, fence, now) { + return mutate(tenantId, fence, now, async (run) => authorityLost(run)); + }, + async finishAuthorityLost(tenantId, fence, now) { + await transaction(async (session) => { + const run = await owned(tenantId, fence, now, session); + if (run.errorCode !== 'channel_authority_changed' || !run.cancellationRequested) + throw new ChannelRunConflict('invalid_transition'); + await lanes().updateOne( + { tenantId, laneId: run.laneId, activeRunId: run.runId }, + { $unset: { activeRunId: 1 } }, + { session }, + ); + const result = await runs().updateOne( + predicate(tenantId, fence, now), + { + $set: { state: 'cancelled', leaseExpiresAt: 0 }, + $unset: terminalUnset, + }, + { session }, + ); + if (result.modifiedCount !== 1) throw new ChannelRunConflict('stale_fence'); + return true; + }); + }, + async readActiveThread(scope, before) { + const { tenantId, channelId } = scope; + const laneId = laneKey(scope); + const lane = await lanes().findOne({ tenantId, channelId, laneId }).lean(); + if (!lane?.activeRunId) return null; + // A delayed event must never stop a later claim/generation, even if that + // run's message was queued before the click. Recovery may conservatively + // reject an old click after re-claim; the user can click Stop again. + return runs() + .findOne({ + tenantId, + channelId, + laneId, + runId: lane.activeRunId, + state: { $in: ['queued', 'executing', 'requires_action'] }, + claimCheckedAt: { $lte: before }, + receivedAt: { $lte: before }, + }) + .lean(); + }, + async requestCancel(scope, runId, observed) { + const { tenantId, channelId } = scope; + if ( + observed && + (!Number.isSafeInteger(observed.epoch) || + observed.epoch < 0 || + !Number.isFinite(observed.before)) + ) + throw new ChannelRunConflict('invalid_transition'); + const result = await runs().updateOne( + { + tenantId, + channelId, + runId, + laneId: laneKey(scope), + state: { $in: ['queued', 'executing', 'requires_action'] }, + ...(observed + ? { + leaseEpoch: observed.epoch, + claimCheckedAt: { $lte: observed.before }, + receivedAt: { $lte: observed.before }, + } + : {}), + }, + { $set: { cancellationRequested: true } }, + ); + return result.matchedCount === 1; + }, + async findDeliveredReply(scope, externalMessageKey) { + const { tenantId, channelId } = scope; + return runs() + .findOne({ + tenantId, + channelId, + laneId: laneKey(scope), + state: 'delivered', + receipts: { $elemMatch: { externalMessageKey, outcome: 'confirmed' } }, + }) + .lean(); + }, + }; +} diff --git a/packages/data-schemas/src/methods/channelSummary.spec.ts b/packages/data-schemas/src/methods/channelSummary.spec.ts new file mode 100644 index 0000000000..e358509610 --- /dev/null +++ b/packages/data-schemas/src/methods/channelSummary.spec.ts @@ -0,0 +1,189 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { ChannelRunRecord } from '~/models/channelRun'; +import type { ChannelSummaryRow } from './channelSummary'; +import { createChannelSummaryStorage, CHANNEL_SUMMARY_MAX_PRIOR_ROWS } from './channelSummary'; +import { runAsSystem, tenantStorage } from '~/config/tenantContext'; +import { createChannelRunModel } from '~/models/channelRun'; + +let mongo: MongoMemoryServer; +const storage = createChannelSummaryStorage(mongoose); +function record(sequence = 1): ChannelRunRecord { + return { + tenantId: 'tenant', + channelId: 'channel', + agentId: 'agent', + laneId: 'lane', + runId: `run-${sequence}`, + sequence, + state: 'delivered', + receivedAt: 1, + payloadHash: 'hash', + leaseEpoch: 1, + leaseExpiresAt: 0, + receipts: [], + cancellationRequested: false, + principal: { + userId: 'same-internal-user', + channelRevision: 'r1', + linkId: 'link', + linkRevision: 'l1', + }, + admission: { + eventId: `event-${sequence}`, + externalChannelId: 'C1', + externalThreadKey: 'C1:123', + externalMessageKey: `C1:${sequence}`, + input: 'secret transcript', + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'ACTOR-A' }, + }, + snapshot: { + status: 'completed', + text: 'secret answer', + elapsedMs: 10, + usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 }, + accounting: { + status: 'available', + currency: 'USD', + amount: '0.1', + basis: 'application_billing', + coverage: 'complete', + reason: 'priced', + calls: [], + }, + }, + }; +} +async function read(scope: ChannelRunRecord): Promise { + return tenantStorage.run({ tenantId: scope.tenantId }, async () => { + const rows: ChannelSummaryRow[] = []; + for await (const row of storage.priorMetrics(scope)) rows.push(row); + return rows; + }); +} +beforeAll(async () => { + mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } }); + await mongoose.connect(mongo.getUri()); + createChannelRunModel(mongoose); + await mongoose.models.ChannelRun.init(); +}, 60000); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => mongoose.models.ChannelRun.deleteMany({})); +}); + +it('reads newest prior metrics only, using the existing lane index, without transcripts', async () => { + await runAsSystem(async () => + mongoose.models.ChannelRun.insertMany([record(1), record(2), record(3)]), + ); + const rows = await read(record(3)); + expect(rows.map((row) => row.sequence)).toEqual([2, 1]); + expect(rows[0]).toEqual({ + sequence: 2, + snapshot: { + status: 'completed', + elapsedMs: 10, + usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 }, + accounting: { + status: 'available', + currency: 'USD', + amount: '0.1', + basis: 'application_billing', + coverage: 'complete', + reason: 'priced', + calls: [], + }, + }, + }); + expect(JSON.stringify(rows)).not.toContain('secret'); + expect(JSON.stringify(rows)).not.toContain('same-internal-user'); + const indexes = await mongoose.models.ChannelRun.collection.indexes(); + expect( + indexes.some( + (index) => + JSON.stringify(index.key) === JSON.stringify({ tenantId: 1, laneId: 1, sequence: 1 }), + ), + ).toBe(true); +}); + +it.each([ + 'tenant', + 'lane', + 'channel', + 'agent', + 'user', + 'channelRevision', + 'link', + 'linkRevision', + 'team', + 'app', + 'actor', + 'thread', + 'externalChannel', +])('excludes a foreign %s even when a malformed row reuses the same lane', async (kind) => { + const foreign = record(1); + if (kind === 'tenant') foreign.tenantId = 'other'; + if (kind === 'lane') foreign.laneId = 'other'; + if (kind === 'channel') foreign.channelId = 'other'; + if (kind === 'agent') foreign.agentId = 'other'; + if (kind === 'user') foreign.principal.userId = 'other'; + if (kind === 'channelRevision') foreign.principal.channelRevision = 'other'; + if (kind === 'link') foreign.principal.linkId = 'other'; + if (kind === 'linkRevision') foreign.principal.linkRevision = 'other'; + if (kind === 'team') foreign.admission.actor.teamId = 'other'; + if (kind === 'app') foreign.admission.actor.appId = 'other'; + if (kind === 'actor') foreign.admission.actor.externalUserId = 'ACTOR-B'; + if (kind === 'thread') foreign.admission.externalThreadKey = 'other'; + if (kind === 'externalChannel') foreign.admission.externalChannelId = 'other'; + await runAsSystem(async () => mongoose.models.ChannelRun.create(foreign)); + expect(await read(record(2))).toEqual([]); +}); + +it('keeps failed/cancelled/legacy attempts visible as metric coverage gaps', async () => { + await runAsSystem(async () => + mongoose.models.ChannelRun.insertMany([ + { ...record(1), state: 'failed', snapshot: undefined }, + { ...record(2), state: 'cancelled' }, + { ...record(3), state: 'blocked' }, + ]), + ); + expect((await read(record(4))).map((row) => row.sequence)).toEqual([3, 2, 1]); +}); + +it('bounds long-history reads at newest 1000 rows without silently altering the sequence cutoff', async () => { + const count = CHANNEL_SUMMARY_MAX_PRIOR_ROWS + 3; + await runAsSystem(async () => + mongoose.models.ChannelRun.insertMany(Array.from({ length: count }, (_, i) => record(i + 1))), + ); + const rows = await read(record(count)); + expect(rows).toHaveLength(CHANNEL_SUMMARY_MAX_PRIOR_ROWS); + expect(rows[0].sequence).toBe(count - 1); + expect(rows[rows.length - 1]?.sequence).toBe(count - CHANNEL_SUMMARY_MAX_PRIOR_ROWS); +}); + +it('throws for invalid sequence and database failure instead of disguising outages as absence', async () => { + await expect(read(record(0))).rejects.toThrow('Invalid channel summary sequence'); + const other = new mongoose.Mongoose(); + await expect( + (async () => { + for await (const row of createChannelSummaryStorage(other).priorMetrics(record(2))) void row; + })(), + ).rejects.toThrow(); +}); + +it('uses the configured history work bound while retaining the newest prior rows', async () => { + await runAsSystem(async () => + mongoose.models.ChannelRun.insertMany(Array.from({ length: 5 }, (_, i) => record(i + 1))), + ); + const configured = createChannelSummaryStorage(mongoose, { maxPriorSummaryRows: 2 }); + const rows = await tenantStorage.run({ tenantId: 'tenant' }, async () => { + const result: ChannelSummaryRow[] = []; + for await (const row of configured.priorMetrics({ ...record(6), sequence: 6 })) + result.push(row); + return result; + }); + expect(rows.map((row) => row.sequence)).toEqual([5, 4]); +}); diff --git a/packages/data-schemas/src/methods/channelSummary.ts b/packages/data-schemas/src/methods/channelSummary.ts new file mode 100644 index 0000000000..3d471e7584 --- /dev/null +++ b/packages/data-schemas/src/methods/channelSummary.ts @@ -0,0 +1,86 @@ +import { channelStorageConfigSchema } from 'librechat-data-provider'; +import type { + ChannelRunSnapshot, + ResponseAccounting, + ChannelStorageConfig, +} from 'librechat-data-provider'; +import type { ChannelRunRecord } from '~/models/channelRun'; + +export type ChannelSummaryScope = Pick< + ChannelRunRecord, + 'tenantId' | 'channelId' | 'agentId' | 'laneId' | 'principal' | 'sequence' | 'admission' +>; + +export interface ChannelSummaryRow { + sequence: number; + snapshot?: Pick & { + accounting?: Pick< + ResponseAccounting, + 'status' | 'currency' | 'coverage' | 'amount' | 'basis' | 'reason' + > & { calls?: Array> }; + }; +} + +export interface ChannelSummaryStorage { + /** Newest prior attempts, with a fixed presentation safety bound; not a billing ledger. */ + priorMetrics(scope: ChannelSummaryScope): AsyncIterable; +} + +/** Bounds footer work independently of thread length. Older rows become explicit coverage gaps. */ +export const CHANNEL_SUMMARY_MAX_PRIOR_ROWS: number = channelStorageConfigSchema.parse( + {}, +).maxPriorSummaryRows; + +export function createChannelSummaryStorage( + mongoose: typeof import('mongoose'), + options: Pick, 'maxPriorSummaryRows'> = {}, +): ChannelSummaryStorage { + const { maxPriorSummaryRows } = channelStorageConfigSchema.parse(options); + return { + async *priorMetrics(scope) { + if (!Number.isSafeInteger(scope.sequence) || scope.sequence < 1) + throw new Error('Invalid channel summary sequence'); + const cursor = mongoose + .model('ChannelRun') + .find({ + tenantId: scope.tenantId, + laneId: scope.laneId, + sequence: { $gte: 1, $lt: scope.sequence }, + channelId: scope.channelId, + agentId: scope.agentId, + 'principal.userId': scope.principal.userId, + 'principal.channelRevision': scope.principal.channelRevision, + 'principal.linkId': scope.principal.linkId ?? null, + 'principal.linkRevision': scope.principal.linkRevision ?? null, + 'admission.actor.teamId': scope.admission.actor.teamId, + 'admission.actor.appId': scope.admission.actor.appId, + 'admission.actor.externalUserId': scope.admission.actor.externalUserId, + 'admission.externalChannelId': scope.admission.externalChannelId, + 'admission.externalThreadKey': scope.admission.externalThreadKey, + }) + .select({ + _id: 0, + sequence: 1, + 'snapshot.status': 1, + 'snapshot.elapsedMs': 1, + 'snapshot.usage': 1, + 'snapshot.accounting.status': 1, + 'snapshot.accounting.currency': 1, + 'snapshot.accounting.coverage': 1, + 'snapshot.accounting.amount': 1, + 'snapshot.accounting.basis': 1, + 'snapshot.accounting.reason': 1, + 'snapshot.accounting.calls.reason': 1, + }) + .sort({ sequence: -1 }) + .limit(maxPriorSummaryRows) + .lean() + .cursor(); + try { + for await (const row of cursor) yield row; + } finally { + await cursor.close(); + } + }, + }; +} diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index f09c8dda87..8792c82e9f 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -5,6 +5,7 @@ export type { PermissionPrincipalRecord, PermissionPrincipalData, } from './provisioning'; +import type { ChannelStorageConfig } from 'librechat-data-provider'; import { createSubagentContentMethods, type SubagentContentMethods } from './subagentContent'; export type { SubagentContentScope, @@ -68,6 +69,20 @@ import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCa /* Agent API Keys */ import { createAgentApiKeyMethods, type AgentApiKeyMethods } from './agentApiKey'; +export { createChannelRunStorage, ChannelRunConflict } from './channelRun'; +export { createChannelSummaryStorage, CHANNEL_SUMMARY_MAX_PRIOR_ROWS } from './channelSummary'; +export type { + ChannelSummaryStorage, + ChannelSummaryScope, + ChannelSummaryRow, +} from './channelSummary'; +export type { + ChannelRunStorage, + ChannelRunEnrollment, + ChannelLaneScope, + ChannelRunRecord, + ChannelLaneRecord, +} from './channelRun'; /* MCP Servers */ import { createMCPServerMethods, type MCPServerMethods } from './mcpServer'; import { createCodeEnvironmentMethods, type CodeEnvironmentMethods } from './codeEnvironment'; @@ -341,6 +356,7 @@ export type AllMethods = PermissionPrincipalStorage & /** Dependencies injected from the api layer into createMethods */ export interface CreateMethodsDeps { + getChannelStorageConfig?: () => Promise>; /** Matches a model name to a canonical key. From @librechat/api. */ matchModelName?: (model: string, endpoint?: string) => string | undefined; /** Finds the first key in values whose key is a substring of model. From @librechat/api. */ @@ -529,7 +545,10 @@ export function createMethods( }; const agentMethods = createAgentMethods(mongoose, agentDeps); return { - ...createUserMethods(mongoose, { getCache: deps.getCache }), + ...createUserMethods(mongoose, { + getCache: deps.getCache, + getChannelStorageConfig: deps.getChannelStorageConfig, + }), ...createToolApprovalGrantMethods(mongoose), ...createSessionMethods(mongoose), ...createTokenMethods(mongoose), diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 78dddcc202..bc2caa057f 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -1,6 +1,7 @@ import mongoose, { FilterQuery } from 'mongoose'; import { AUTH_USER_DOC_MAX_AGE_MS } from 'librechat-data-provider'; import type { + ChannelStorageConfig, BalanceRefillMode, RefillIntervalUnit, StatefulCodeEnvironment, @@ -16,6 +17,10 @@ import type { } from '~/types'; import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types'; import type { CacheStore } from '~/types'; +import { + eraseChannelPersistenceForUser, + resolveChannelErasureConfig, +} from '~/methods/channelErasure'; import { closeResponseScopes, eraseResponseRecords } from './responseScope'; import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search'; import { evictAuthUserDocsForUsers } from '~/utils/eviction'; @@ -35,6 +40,7 @@ const TWO_FACTOR_ENROLLMENT_PROJECTION = '+totpSecret +backupCodes +pendingTotpSecret +pendingBackupCodes +twoFactorAcknowledgementNonceHash +twoFactorFinalizationNonceHash'; interface UserMethodDeps { + getChannelStorageConfig?: () => Promise>; getCache?: (key: string) => CacheStore | undefined; /** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */ delay?: (ms: number) => Promise; @@ -644,8 +650,18 @@ export function createUserMethods( const User = mongoose.models.User; const owner = await User.findById(userId).select('tenantId').lean>(); if (owner) { + const channelConfig = resolveChannelErasureConfig( + (await deps.getChannelStorageConfig?.()) ?? {}, + ); await closeResponseScopes(mongoose, { userId, tenantId: owner.tenantId }); await eraseResponseRecords(mongoose, { userId, tenantId: owner.tenantId }); + // Admission writes the same owner anchor in its insertion transaction. + // Closing it therefore precedes both this sweep and every delayed retry. + await eraseChannelPersistenceForUser( + mongoose, + { userId, tenantId: owner.tenantId }, + channelConfig, + ); } await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); const result = await User.deleteOne({ _id: userId }); diff --git a/packages/data-schemas/src/models/channelRun.ts b/packages/data-schemas/src/models/channelRun.ts new file mode 100644 index 0000000000..86d4ecd4e3 --- /dev/null +++ b/packages/data-schemas/src/models/channelRun.ts @@ -0,0 +1,179 @@ +import { Schema } from 'mongoose'; +import type { + ChannelRunView, + ChannelRunPrincipal, + ChannelDeliveryReceipt, +} from 'librechat-data-provider'; +import type { Model } from 'mongoose'; +import { applyTenantIsolation } from './plugins/tenantIsolation'; + +export interface ChannelRunRecord extends Omit { + tenantId: string; + agentId: string; + laneId: string; + principal: ChannelRunPrincipal; + payloadHash: string; + receivedAt: number; + leaseOwner?: string; + leaseEpoch: number; + leaseExpiresAt: number; + /** Durable fairness stamp; not execution authority or a retry policy. */ + claimCheckedAt?: number; + /** Persisted before dispatch; every retry attaches to this exact identity. */ + execution?: { + protocol?: 'responses'; + responseId?: string; + apiKeyId?: string; + encryptedApiKey?: string; + requestId: string; + conversationId: string; + parentMessageId: string; + generationCreatedAt?: number; + deliveryKey?: string; + enqueueAttemptedAt?: number; + }; + errorCode?: string; + deliveredAt?: number; +} + +export interface ChannelLaneRecord { + tenantId: string; + userId: string; + channelId: string; + laneId: string; + nextSequence: number; + activeRunId?: string; + conversationId?: string; + responseId?: string; +} + +export function createChannelRunModel( + mongoose: typeof import('mongoose'), +): Model { + const receipt = new Schema( + { + operationId: { type: String, required: true }, + outcome: { + type: String, + required: true, + enum: ['planned', 'inflight', 'confirmed', 'retryable', 'unknown', 'permanent_failure'], + }, + attempts: { type: Number, required: true }, + externalMessageKey: String, + externalFileId: String, + retryAt: Number, + errorCode: String, + upload: { + type: new Schema( + { + fileId: { type: String, required: true }, + stage: { + type: String, + required: true, + enum: ['allocated', 'uploaded'], + }, + }, + { _id: false }, + ), + }, + }, + { _id: false }, + ); + const schema = new Schema( + { + tenantId: { type: String, required: true }, + channelId: { type: String, required: true }, + agentId: { type: String, required: true }, + laneId: { type: String, required: true }, + runId: { type: String, required: true }, + state: { + type: String, + required: true, + enum: [ + 'queued', + 'executing', + 'requires_action', + 'delivering', + 'delivered', + 'failed', + 'cancelled', + 'blocked', + ], + }, + admission: { type: Schema.Types.Mixed, required: true }, + principal: { type: Schema.Types.Mixed, required: true }, + payloadHash: { type: String, required: true }, + sequence: { type: Number, required: true }, + receivedAt: { type: Number, required: true }, + leaseOwner: String, + leaseEpoch: { type: Number, required: true, default: 0 }, + leaseExpiresAt: { type: Number, required: true, default: 0 }, + claimCheckedAt: { type: Number, default: 0 }, + execution: Schema.Types.Mixed, + snapshot: Schema.Types.Mixed, + progress: Schema.Types.Mixed, + plan: Schema.Types.Mixed, + receipts: { type: [receipt], required: true, default: [] }, + cancellationRequested: { type: Boolean, required: true, default: false }, + errorCode: String, + deliveredAt: Number, + }, + { timestamps: true, minimize: false }, + ); + schema.index({ tenantId: 1, runId: 1 }, { unique: true }); + schema.index({ tenantId: 1, 'principal.userId': 1 }); + schema.index( + { + tenantId: 1, + channelId: 1, + 'admission.actor.teamId': 1, + 'admission.actor.appId': 1, + 'admission.eventId': 1, + }, + { unique: true, name: 'channel_run_event_v2' }, + ); + schema.index({ tenantId: 1, laneId: 1, sequence: 1 }, { unique: true }); + schema.index( + { tenantId: 1, laneId: 1, state: 1, sequence: 1 }, + { name: 'channel_run_active_predecessor' }, + ); + schema.index({ tenantId: 1, channelId: 1, state: 1, leaseExpiresAt: 1, receivedAt: 1 }); + schema.index( + { + tenantId: 1, + channelId: 1, + state: 1, + claimCheckedAt: 1, + receivedAt: 1, + sequence: 1, + leaseExpiresAt: 1, + }, + { name: 'channel_run_fair_claim' }, + ); + schema.index({ state: 1, _id: 1 }); + schema.index({ tenantId: 1, channelId: 1, 'receipts.externalMessageKey': 1 }); + applyTenantIsolation(schema); + return mongoose.models.ChannelRun ?? mongoose.model('ChannelRun', schema); +} + +export function createChannelLaneModel( + mongoose: typeof import('mongoose'), +): Model { + const schema = new Schema( + { + tenantId: { type: String, required: true }, + userId: { type: String, required: true }, + channelId: { type: String, required: true }, + laneId: { type: String, required: true }, + nextSequence: { type: Number, required: true, default: 0 }, + activeRunId: String, + conversationId: String, + responseId: String, + }, + { timestamps: true }, + ); + schema.index({ tenantId: 1, laneId: 1 }, { unique: true }); + schema.index({ tenantId: 1, userId: 1 }); + applyTenantIsolation(schema); + return mongoose.models.ChannelLane ?? mongoose.model('ChannelLane', schema); +} diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts index badff5d2cc..29cef2b30f 100644 --- a/packages/data-schemas/src/models/index.ts +++ b/packages/data-schemas/src/models/index.ts @@ -1,5 +1,6 @@ import { createAgentQueuedTurnModel, createAgentQueuedTurnSequenceModel } from './queuedTurn'; +import { createChannelRunModel, createChannelLaneModel } from './channelRun'; import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence'; import { createScheduleModel, createScheduleRunModel } from './schedule'; import { getTenantIndexMigrationHint } from '~/migrations/tenantIndexes'; @@ -75,6 +76,8 @@ export function createModels(mongoose: typeof import('mongoose')): { SubagentContent: ReturnType; Agent: ReturnType; AgentApiKey: ReturnType; + ChannelRun: ReturnType; + ChannelLane: ReturnType; AgentCategory: ReturnType; MCPServer: ReturnType; Role: ReturnType; @@ -132,6 +135,8 @@ export function createModels(mongoose: typeof import('mongoose')): { SubagentContent: createSubagentContentModel(mongoose), Agent: createAgentModel(mongoose), AgentApiKey: createAgentApiKeyModel(mongoose), + ChannelRun: createChannelRunModel(mongoose), + ChannelLane: createChannelLaneModel(mongoose), AgentCategory: createAgentCategoryModel(mongoose), MCPServer: createMCPServerModel(mongoose), Role: createRoleModel(mongoose), diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.ts index 0b7329d322..3c54e3f526 100644 --- a/packages/data-schemas/src/schema/user.ts +++ b/packages/data-schemas/src/schema/user.ts @@ -1,6 +1,7 @@ import { Schema } from 'mongoose'; import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider'; import { applySearchTokens } from '~/models/plugins/searchTokens'; +import { isReservedTenantId } from '~/config/tenantContext'; import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search'; import { IUser } from '~/types'; @@ -232,6 +233,10 @@ const userSchema: Schema = new Schema( tenantId: { type: String, index: true, + validate: { + validator: (value: string | undefined) => !isReservedTenantId(value), + message: 'Reserved tenant identifier', + }, }, }, { timestamps: true },