📬 feat: Persist Channel Events and Delivery Receipts (#17595)

* 📬 feat: Persist Channel Events and Delivery Receipts

* fix: enforce channel run identity and fenced recovery

* fix: fence and erase channel state during account deletion

* refactor: share channel owner admission with private persistence

* refactor: name channel account-erasure boundary explicitly

* fix: prove channel identity and delivery authority at storage boundaries

* refactor: enforce channel delivery protocol and terminal lifecycle

* fix: enforce channel tenant and operation boundaries

* fix: initialize channel config from its composition owner

* test: verify config installs channel storage reader

* test: complete config initializer contract in model mocks

* test: include channel config reader in migration fixture

* test: await SSE subscription before advancing keepalive time

* test: Observe and Drain Concurrent Fixture Work
This commit is contained in:
Dustin Healy 2026-11-10 21:03:57 -08:00 • committed by GitHub
parent 1458e841e4
commit 7dd4fafdf3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
48 changed files with 3816 additions and 54 deletions

View file

@ -5,16 +5,19 @@ const {
findMatchingPattern,
isDeploymentSkillId,
createMessageBudgetReader,
createChannelStorageConfigReader,
} = require('@librechat/api');
const getLogStores = require('~/cache/getLogStores');
const messageBudget = createMessageBudgetReader();
const channelStorageConfig = createChannelStorageConfigReader();
const methods = createMethods(mongoose, {
matchModelName,
findMatchingPattern,
isExternalSkillId: isDeploymentSkillId,
getCache: getLogStores,
getChannelStorageConfig: channelStorageConfig.getConfig,
getMCPAppMessageBudget: messageBudget.getBudget,
});
@ -28,5 +31,6 @@ const seedDatabase = async () => {
module.exports = {
...methods,
initializeMessageBudget: messageBudget.initialize,
initializeChannelStorageConfig: channelStorageConfig.initialize,
seedDatabase,
};

View file

@ -3,15 +3,21 @@ jest.mock('@librechat/api', () => ({
matchModelName: jest.fn(),
findMatchingPattern: jest.fn(),
isDeploymentSkillId: jest.fn(),
createChannelStorageConfigReader: jest.fn(() => ({
getConfig: jest.fn(),
initialize: jest.fn(),
})),
createMessageBudgetReader: jest.fn(() => ({ getBudget: jest.fn(), initialize: jest.fn() })),
}));
jest.mock('~/cache/getLogStores', () => jest.fn());
const { createMethods } = require('@librechat/data-schemas');
const { createMessageBudgetReader } = require('@librechat/api');
const { initializeMessageBudget } = require('./index');
const { createMessageBudgetReader, createChannelStorageConfigReader } = require('@librechat/api');
const { initializeMessageBudget, initializeChannelStorageConfig } = require('./index');
const reader = createMessageBudgetReader.mock.results[0].value;
const { getMCPAppMessageBudget } = createMethods.mock.calls[0][1];
const channelStorageConfigReader = createChannelStorageConfigReader.mock.results[0].value;
const { getMCPAppMessageBudget, getChannelStorageConfig: channelStorageConfigDependency } =
createMethods.mock.calls[0][1];
describe('message App budget dependency wiring', () => {
it('passes the same reader to storage and initializer to the composition root', () => {
@ -20,3 +26,11 @@ describe('message App budget dependency wiring', () => {
expect(reader.getBudget).not.toHaveBeenCalled();
});
});
describe('channel storage configuration dependency wiring', () => {
it('injects the deployment reader without reading config during module initialization', () => {
expect(channelStorageConfigDependency).toBe(channelStorageConfigReader.getConfig);
expect(initializeChannelStorageConfig).toBe(channelStorageConfigReader.initialize);
expect(channelStorageConfigReader.getConfig).not.toHaveBeenCalled();
});
});

View file

@ -35,6 +35,7 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
saveMessage: jest.fn(async (req, message) => message),
getConvo: jest.fn(async () => null),
getMessages: jest.fn(async () => []),

View file

@ -3,6 +3,7 @@ const mockGetCacheMultiplier = jest.fn(() => 1);
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
getMultiplier: (...args) => mockGetMultiplier(...args),
getCacheMultiplier: (...args) => mockGetCacheMultiplier(...args),
}));

View file

@ -2,6 +2,7 @@ const mockSpendTokens = jest.fn().mockResolvedValue(undefined);
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
spendTokens: mockSpendTokens,
}));

View file

@ -33,6 +33,7 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
saveMessage: jest.fn(async (req, message) => message),
getConvo: jest.fn(async () => null),
getMessages: jest.fn(async () => []),

View file

@ -20,6 +20,7 @@ const mockRecordCollectedUsage = jest
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
spendTokens: (...args) => mockSpendTokens(...args),
spendStructuredTokens: (...args) => mockSpendStructuredTokens(...args),
getMultiplier: mockGetMultiplier,

View file

@ -51,6 +51,7 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
saveMessage: (...args) => mockSaveMessage(...args),
getPersistedPrivateTextId: async (...args) =>
(await mockHasPersistedPrivateText(...args)) ? 'protected-row-id' : null,

View file

@ -39,6 +39,7 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
saveMessage: (...args) => mockSaveMessage(...args),
}));
@ -86,6 +87,28 @@ function mockSubscribeSuccess() {
});
}
/** A real HTTP socket must reach subscribe before fake time can exercise its keepalive. */
function openSilentStream() {
let resolveAttached;
const subscribed = new Promise((resolve) => {
resolveAttached = resolve;
});
mockGenerationJobManager.subscribe.mockImplementation(async (_streamId, _writeEvent, onDone) => {
resolveAttached(onDone);
return { unsubscribe: jest.fn() };
});
const pending = request(app)
.get('/agents/chat/stream/stream-123')
.then((res) => res);
const attached = Promise.race([
subscribed,
pending.then((res) => {
throw new Error(`Stream ended before subscribing: HTTP ${res.status}`);
}),
]);
return { pending, attached };
}
describe('SSE stream tenant isolation', () => {
beforeEach(() => {
jest.clearAllMocks();
@ -411,19 +434,8 @@ describe('SSE stream tenant isolation', () => {
status: 'running',
createdAt: 1000,
});
let finish;
mockGenerationJobManager.subscribe.mockImplementation(
async (_streamId, _writeEvent, onDone) => {
finish = onDone;
return { unsubscribe: jest.fn() };
},
);
const pending = request(app)
.get('/agents/chat/stream/stream-123')
.then((res) => res);
await jest.advanceTimersByTimeAsync(0);
await new Promise((resolve) => setImmediate(resolve));
const { pending, attached } = openSilentStream();
const finish = await attached;
await jest.advanceTimersByTimeAsync(60_000);
finish({ final: true });
const res = await pending;
@ -470,19 +482,8 @@ describe('SSE stream tenant isolation', () => {
status: 'running',
createdAt: 1000,
});
let finish;
mockGenerationJobManager.subscribe.mockImplementation(
async (_streamId, _writeEvent, onDone) => {
finish = onDone;
return { unsubscribe: jest.fn() };
},
);
const pending = request(app)
.get('/agents/chat/stream/stream-123')
.then((res) => res);
await jest.advanceTimersByTimeAsync(0);
await new Promise((resolve) => setImmediate(resolve));
const { pending, attached } = openSilentStream();
const finish = await attached;
await jest.advanceTimersByTimeAsync(60_000);
finish({ final: true });
const res = await pending;
@ -502,19 +503,8 @@ describe('SSE stream tenant isolation', () => {
status: 'running',
createdAt: 1000,
});
let finish;
mockGenerationJobManager.subscribe.mockImplementation(
async (_streamId, _writeEvent, onDone) => {
finish = onDone;
return { unsubscribe: jest.fn() };
},
);
const pending = request(app)
.get('/agents/chat/stream/stream-123')
.then((res) => res);
await jest.advanceTimersByTimeAsync(0);
await new Promise((resolve) => setImmediate(resolve));
const { pending, attached } = openSilentStream();
const finish = await attached;
await jest.advanceTimersByTimeAsync(60_000);
expect(mockGenerationJobManager.subscribe).toHaveBeenCalled();

View file

@ -17,6 +17,7 @@ jest.mock('~/server/services/start/tools', () => ({
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
getApplicableConfigs: jest.fn().mockResolvedValue([]),
getUserPrincipals: jest.fn().mockResolvedValue([]),
}));

View file

@ -22,6 +22,7 @@ jest.mock('@librechat/data-schemas', () => {
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
getApplicableConfigs: jest.fn().mockResolvedValue([]),
getUserPrincipals: jest.fn().mockResolvedValue([]),
}));
@ -61,14 +62,16 @@ jest.mock('@librechat/api', () => ({
// ── Tests ──────────────────────────────────────────────────────────────
const { getAppConfig, invalidateConfigCaches } = require('../app');
const { initializeMessageBudget } = require('~/models');
const { initializeMessageBudget, initializeChannelStorageConfig } = require('~/models');
// Capture module-composition calls before clearMocks resets the spy history.
const budgetInitializations = [...initializeMessageBudget.mock.calls];
const channelInitializations = [...initializeChannelStorageConfig.mock.calls];
const invalidationOptions = mockCreateConfigCacheInvalidation.mock.calls[0][0];
const pubSubOptions = mockCreateRedisPubSub.mock.calls[0][0];
test('Config supplies its actual reader exactly once during composition', () => {
expect(budgetInitializations).toEqual([[getAppConfig]]);
expect(channelInitializations).toEqual([[getAppConfig]]);
});
describe('invalidateConfigCaches', () => {

View file

@ -88,6 +88,7 @@ const {
// Config owns the reader; models never import this module to obtain it.
db.initializeMessageBudget(getAppConfig);
db.initializeChannelStorageConfig(getAppConfig);
/**
* Admin config writes clear the per-principal override caches, the MCP config-source servers

View file

@ -3,7 +3,11 @@ jest.mock('@librechat/api', () => ({
fetchModels: jest.fn(),
}));
jest.mock('./app');
jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn() }));
jest.mock('~/models', () => ({
getUserKeyValues: jest.fn(),
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
}));
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
logger: { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), info: jest.fn() },

View file

@ -14,6 +14,7 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('~/models', () => ({
getUserKeyValues: jest.fn(),
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
}));
const exampleConfig = {

View file

@ -28,6 +28,7 @@ const { ImportBatchBuilder } = require('./importBatchBuilder');
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
bulkIncrementTagCounts: jest.fn(),
bulkSaveConvos: jest.fn(),
bulkSaveMessages: jest.fn(),

View file

@ -8,6 +8,7 @@ const importConversations = require('./importConversations');
jest.mock('~/models', () => ({
initializeMessageBudget: jest.fn(),
initializeChannelStorageConfig: jest.fn(),
bulkIncrementTagCounts: jest.fn(),
bulkSaveConvos: jest.fn(),
bulkSaveMessages: jest.fn(),

View file

@ -1,6 +1,10 @@
jest.mock('../connect', () => jest.fn().mockResolvedValue(true));
jest.mock('@librechat/api', () => ({
ensureRequiredCollectionsExist: jest.fn().mockResolvedValue(undefined),
createChannelStorageConfigReader: jest.fn(() => ({
initialize: jest.fn(),
getConfig: jest.fn(),
})),
createMessageBudgetReader: jest.fn(() => ({ initialize: jest.fn(), getBudget: jest.fn() })),
matchModelName: jest.fn(),
findMatchingPattern: jest.fn(),

View file

@ -0,0 +1,35 @@
import type { AppConfig } from '@librechat/data-schemas';
import { createChannelStorageConfigReader } from './config';
it('fails closed until the configuration service installs its reader', async () => {
await expect(createChannelStorageConfigReader().getConfig()).rejects.toThrow(
'not been initialized',
);
});
it('reads live base deployment bounds at each operation boundary', async () => {
const config = { endpoints: { agents: { channels: { erasureBatchSize: 2 } } } };
const read = jest.fn(async () => config as AppConfig);
const effective = createChannelStorageConfigReader();
effective.initialize(read);
expect(await effective.getConfig()).toMatchObject({ erasureBatchSize: 2, claimBatchSize: 25 });
config.endpoints.agents.channels.erasureBatchSize = 3;
expect((await effective.getConfig()).erasureBatchSize).toBe(3);
expect(read).toHaveBeenCalledTimes(2);
expect(read).toHaveBeenLastCalledWith({ baseOnly: true });
effective.initialize(async () => ({}) as AppConfig);
expect((await effective.getConfig()).erasureBatchSize).toBe(250);
});
it('rejects invalid bounds and propagates config outages', async () => {
const effective = createChannelStorageConfigReader();
effective.initialize(
async () => ({ endpoints: { agents: { channels: { erasureBatchSize: 0 } } } }) as AppConfig,
);
await expect(effective.getConfig()).rejects.toThrow();
const failure = new Error('config unavailable');
effective.initialize(async () => {
throw failure;
});
await expect(effective.getConfig()).rejects.toBe(failure);
});

View file

@ -0,0 +1,22 @@
import { channelStorageConfigSchema } from 'librechat-data-provider';
import type { ChannelStorageConfig } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type { GetAppConfigOptions } from '~/app/service';
/** Config installs its reader during composition, without a models-to-config import cycle. */
export function createChannelStorageConfigReader(): {
initialize: (reader: (options?: GetAppConfigOptions) => Promise<AppConfig>) => void;
getConfig: () => Promise<ChannelStorageConfig>;
} {
let readConfig: ((options?: GetAppConfigOptions) => Promise<AppConfig>) | undefined;
return {
initialize(reader) {
readConfig = reader;
},
async getConfig() {
if (!readConfig) throw new Error('Channel storage config reader has not been initialized');
const config = await readConfig({ baseOnly: true });
return channelStorageConfigSchema.parse(config.endpoints?.agents?.channels ?? {});
},
};
}

View file

@ -155,3 +155,5 @@ export type * from './types';
export * from './tools/approval';
export * from './mcp/approval';
export { createChannelStorageConfigReader } from './channels/config';

View file

@ -574,3 +574,22 @@ describe('requirePreAuthTenantContext', () => {
expect(next).not.toHaveBeenCalled();
});
});
describe('reserved tenant partition', () => {
it('rejects a reserved tenant restored from the authenticated user', () => {
const req = mockReq({ tenantId: '_', role: 'user' });
const res = mockRes();
const next = jest.fn();
tenantContextMiddleware(req, res, next);
expect(res.status).toHaveBeenCalledWith(403);
expect(next).not.toHaveBeenCalled();
});
it('rejects the reserved tenant when a request resumes after an async boundary', async () => {
const req = mockReq({ tenantId: '_', role: 'user' });
const res = mockRes();
const next = jest.fn();
await restoreTenantContextFromReq(req, res, next);
expect(res.status).toHaveBeenCalledWith(403);
expect(next).not.toHaveBeenCalled();
});
});

View file

@ -184,3 +184,25 @@ describe('preAuthTenantMiddleware', () => {
);
});
});
describe('reserved tenant header boundary', () => {
it('rejects the tenantless persistence sentinel before invoking unauthenticated handlers', () => {
const previous = process.env.TRUST_TENANT_HEADER;
process.env.TRUST_TENANT_HEADER = 'true';
const response = { status: jest.fn().mockReturnThis(), json: jest.fn() };
const next = jest.fn();
try {
preAuthTenantMiddleware(
{ headers: { 'x-tenant-id': '_' } } as unknown as Request,
response as unknown as Response,
next,
);
expect(response.status).toHaveBeenCalledWith(400);
expect(response.json).toHaveBeenCalledWith({ error: 'Reserved tenant identifier' });
expect(next).not.toHaveBeenCalled();
} finally {
if (previous === undefined) delete process.env.TRUST_TENANT_HEADER;
else process.env.TRUST_TENANT_HEADER = previous;
}
});
});

View file

@ -50,6 +50,12 @@ export function preAuthTenantMiddleware(req: Request, res: Response, next: NextF
}
const rejection = rejectTenantHeader(tenantId);
if (rejection === 'reserved') {
runWithTenantContext(requestContext, () =>
res.status(400).json({ error: 'Reserved tenant identifier' }),
);
return;
}
if (rejection === 'system') {
runWithTenantContext(requestContext, () => {
logger.warn('[preAuthTenant] Rejected __SYSTEM__ sentinel in X-Tenant-Id header', {

View file

@ -1,7 +1,13 @@
import { randomUUID } from 'crypto';
import { unlink } from 'fs/promises';
import { isMainThread } from 'worker_threads';
import { tenantStorage, getTenantId, logger, SYSTEM_TENANT_ID } from '@librechat/data-schemas';
import {
tenantStorage,
getTenantId,
logger,
SYSTEM_TENANT_ID,
isReservedTenantId,
} from '@librechat/data-schemas';
import type { Response, NextFunction, RequestHandler } from 'express';
import type { TenantContext } from '@librechat/data-schemas';
import type { ServerRequest } from '~/types/http';
@ -171,11 +177,15 @@ export function tenantContextMiddleware(
const context = buildTenantContext(req);
const { tenantId } = context;
if (tenantId === SYSTEM_TENANT_ID) {
if (tenantId === SYSTEM_TENANT_ID || isReservedTenantId(tenantId)) {
logger.warn('[tenantContextMiddleware] Rejected system tenant for request route', {
path: req.path,
});
res.status(403).json({ error: SYSTEM_TENANT_REJECTION_MESSAGE });
res.status(403).json({
error: isReservedTenantId(tenantId)
? 'Reserved tenant identifier'
: SYSTEM_TENANT_REJECTION_MESSAGE,
});
return;
}
@ -305,11 +315,17 @@ export function restoreTenantContextFromReq(
return;
}
if (resolvedTenantId === SYSTEM_TENANT_ID) {
if (resolvedTenantId === SYSTEM_TENANT_ID || isReservedTenantId(resolvedTenantId)) {
logger.warn('[restoreTenantContextFromReq] Rejected system tenant for request route', {
path: req.path,
});
return rejectRequestWithUploadCleanup(req, res, SYSTEM_TENANT_REJECTION_MESSAGE);
return rejectRequestWithUploadCleanup(
req,
res,
isReservedTenantId(resolvedTenantId)
? 'Reserved tenant identifier'
: SYSTEM_TENANT_REJECTION_MESSAGE,
);
}
const currentContext = tenantStorage.getStore();

View file

@ -0,0 +1,7 @@
import { rejectTenantHeader } from './tenantHeader';
it('reserves the channel tenantless identifier without banning underscores inside real tenant IDs', () => {
expect(rejectTenantHeader('_')).toBe('reserved');
expect(rejectTenantHeader('org_1')).toBeUndefined();
expect(rejectTenantHeader('__SYSTEM__')).toBe('system');
});

View file

@ -1,16 +1,19 @@
import { SYSTEM_TENANT_ID } from '@librechat/data-schemas';
import { SYSTEM_TENANT_ID, isReservedTenantId } from '@librechat/data-schemas';
const MAX_TENANT_ID_LENGTH = 128;
const VALID_TENANT_ID = /^[-a-zA-Z0-9_.]+$/;
/**
* Why a trusted `X-Tenant-Id` value names no tenant: the system sentinel, or a value that is
* Why a trusted `X-Tenant-Id` value names no tenant: a reserved persistence/system sentinel, or a value that is
* too long or carries characters outside the tenant id alphabet. `undefined` accepts it.
*/
export function rejectTenantHeader(tenantId: string): 'system' | 'malformed' | undefined {
export function rejectTenantHeader(
tenantId: string,
): 'system' | 'reserved' | 'malformed' | undefined {
if (tenantId === SYSTEM_TENANT_ID) {
return 'system';
}
if (isReservedTenantId(tenantId)) return 'reserved';
if (tenantId.length > MAX_TENANT_ID_LENGTH || !VALID_TENANT_ID.test(tenantId)) {
return 'malformed';
}

View file

@ -3371,3 +3371,33 @@ describe('directory verification deadline configuration', () => {
);
});
});
describe('channel storage work limit', () => {
it('preserves the claim default and exposes an operator override through YAML config', () => {
expect(agentsEndpointSchema.parse({ channels: {} }).channels?.claimBatchSize).toBe(25);
expect(
configSchema.parse({
version: '1.3.17',
endpoints: { agents: { channels: { claimBatchSize: 60 } } },
}).endpoints?.agents?.channels?.claimBatchSize,
).toBe(60);
});
it.each([0, 101, 1.5, '25'])('rejects invalid claim batch %s', (claimBatchSize) => {
expect(agentsEndpointSchema.safeParse({ channels: { claimBatchSize } }).success).toBe(false);
});
});
describe('channel erasure configuration', () => {
it('defaults cleanup batches and exposes a deployment override', () => {
expect(agentsEndpointSchema.parse({ channels: {} }).channels?.erasureBatchSize).toBe(250);
expect(
configSchema.parse({
version: '1.3.17',
endpoints: { agents: { channels: { erasureBatchSize: 12 } } },
}).endpoints?.agents?.channels?.erasureBatchSize,
).toBe(12);
});
it.each([0, 1001, 1.5, '250'])('rejects invalid erasure batch %s', (erasureBatchSize) => {
expect(agentsEndpointSchema.safeParse({ channels: { erasureBatchSize } }).success).toBe(false);
});
});

View file

@ -1229,6 +1229,25 @@ const managementApiAuthSchema = z
}
});
export const CHANNEL_DELIVERY_PARTS_HARD_MAX = 100;
/** Storage work per claim transaction; shared by configuration and injected storage defaults. */
export const channelStorageConfigSchema = z.object({
erasureBatchSize: z.number().int().min(1).max(1000).default(250),
claimBatchSize: z.number().int().min(1).max(100).default(25),
maxDeliveryParts: z
.number()
.int()
.min(1)
.max(CHANNEL_DELIVERY_PARTS_HARD_MAX)
.default(CHANNEL_DELIVERY_PARTS_HARD_MAX),
maxMetadataPageSize: z.number().int().min(1).max(100).default(100),
projectionYieldMs: z.number().int().min(100).max(60_000).default(1000),
maxPriorSummaryRows: z.number().int().min(1).max(10_000).default(1000),
});
export type ChannelStorageConfig = z.infer<typeof channelStorageConfigSchema>;
const managementApiSchema = z.object({ auth: managementApiAuthSchema.optional() }).strict();
/**
@ -2260,6 +2279,7 @@ export const agentsEndpointSchema = baseEndpointSchema
.optional(),
managementApi: managementApiSchema.optional(),
remoteApi: remoteApiSchema.optional(),
channels: channelStorageConfigSchema.optional(),
/** Human-in-the-loop tool approval policy. Off by default. */
toolApproval: toolApprovalPolicySchema,
/** Ask User question behavior; see {@link askUserQuestionConfigSchema}. */

View file

@ -115,4 +115,6 @@ export * from './approval';
export * from './types/responseReview';
export * from './types/channelRuns';
export * from './codeSetup';

View file

@ -0,0 +1,162 @@
import type { AccountingBasis, ResponseAccounting } from './responseAccounting';
/** Verified transport identities are supplied by the gateway, never by Agent output. */
export interface ChannelRunActor {
teamId: string;
appId: string;
externalUserId: string;
}
export interface ChannelRunPrincipal {
userId: string;
channelRevision: string;
linkId?: string;
linkRevision?: string;
}
export interface ChannelRunAdmission {
eventId: string;
externalChannelId: string;
externalThreadKey: string;
externalMessageKey: string;
actor: ChannelRunActor;
input: string;
}
export type ChannelRunState =
| 'queued'
| 'executing'
| 'requires_action'
| 'delivering'
| 'delivered'
| 'failed'
| 'cancelled'
| 'blocked';
export interface ChannelRunFence {
runId: string;
owner: string;
epoch: number;
}
export interface ChannelRunUsage {
inputTokens: number;
outputTokens: number;
totalTokens: number;
cachedTokens?: number;
reasoningTokens?: number;
}
export interface ChannelSummaryMetric<T> {
coverage: 'complete' | 'partial' | 'unavailable';
coveredRuns: number;
/** Present only when every expected run is covered. */
value?: T;
/** A known subtotal, never presented as the full thread total. */
knownValue?: T;
}
/** Current run plus prior attempts in this actor/link revision's lane, not other thread actors. */
export interface ChannelRunSummary {
version: 1;
scope: 'actor_lane';
throughSequence: number;
includesCurrent: true;
priorRuns: number;
expectedRuns: number;
observedRuns: number;
costUSD: ChannelSummaryMetric<string> & { basis?: AccountingBasis };
/** Sum of per-run admission-to-final durations, not the thread's wall-clock span. */
elapsedMs: ChannelSummaryMetric<number>;
tokens: ChannelSummaryMetric<
Pick<ChannelRunUsage, 'inputTokens' | 'outputTokens' | 'totalTokens'>
>;
/** Existing authorized trace UI session; never a collector URL or a raw OAuth URL. */
session?: { url: string };
}
/** This is a safe projection, not the Agent transcript or private reasoning. */
export interface ChannelRunSnapshot {
recovery?: 'execution_state_unavailable';
responseId?: string;
text: string;
status: 'running' | 'requires_action' | 'completed' | 'failed' | 'cancelled';
elapsedMs: number;
usage?: ChannelRunUsage;
accounting?: ResponseAccounting;
summary?: ChannelRunSummary;
artifacts?: ChannelArtifact[];
artifactsUnavailable?: boolean;
activity?: 'working' | 'tool_requested' | 'tool_finished';
pendingAction?: {
actionId: string;
kind: 'tool_approval' | 'ask_user_question' | 'oauth';
expiresAt: number;
};
}
export interface ChannelArtifact {
artifactId: string;
filename: string;
mediaType: string;
bytes: number;
sha256: string;
}
/** A deterministic render plan is immutable once its first transport operation starts. */
export interface ChannelDeliveryPart {
operationId: string;
kind: 'message' | 'artifact';
required: boolean;
text: string;
/** Provider-defined, versioned JSON presentation; immutable alongside the text fallback. */
presentation?: { format: string; payload: string };
/** Opaque server-authorized artifact identity; never an arbitrary URL or disk path. */
artifactId?: string;
}
export interface ChannelDeliveryPlan {
revision: string;
rendererVersion: string;
parts: ChannelDeliveryPart[];
}
export type ChannelDeliveryOutcome =
| 'planned'
| 'inflight'
| 'confirmed'
| 'retryable'
| 'unknown'
| 'permanent_failure';
export interface ChannelDeliveryReceipt {
operationId: string;
outcome: ChannelDeliveryOutcome;
attempts: number;
externalMessageKey?: string;
externalFileId?: string;
retryAt?: number;
errorCode?: string;
upload?: { fileId: string; stage: 'allocated' | 'uploaded' };
}
export interface ChannelRunProgress
extends Omit<ChannelDeliveryReceipt, 'operationId' | 'externalFileId'> {
revision: number;
rendererVersion: string;
text: string;
}
export interface ChannelRunView {
runId: string;
channelId: string;
revision: string;
state: ChannelRunState;
admission: ChannelRunAdmission;
sequence: number;
snapshot?: ChannelRunSnapshot;
progress?: ChannelRunProgress;
plan?: ChannelDeliveryPlan;
receipts: ChannelDeliveryReceipt[];
cancellationRequested: boolean;
}

View file

@ -50,3 +50,14 @@ describe('scopedCacheKey', () => {
);
});
});
describe('reserved channel partition', () => {
it('refuses to create a named-tenant cache namespace for the tenantless sentinel', () => {
expect(() => tenantStorage.run({ tenantId: '_' }, () => scopedCacheKey('private'))).toThrow(
'Reserved tenant identifier',
);
expect(tenantStorage.run({ tenantId: 'org_1' }, () => scopedCacheKey('private'))).toBe(
'private:org_1',
);
});
});

View file

@ -11,6 +11,11 @@ export interface TenantContext {
/** Sentinel value for deliberate cross-tenant system operations */
export const SYSTEM_TENANT_ID = '__SYSTEM__';
/** Reserved for persisted tenantless channel partitions; never an authenticated tenant. */
export function isReservedTenantId(tenantId: string | undefined): boolean {
return tenantId === '_';
}
/**
* AsyncLocalStorage instance for propagating tenant context.
* Callbacks passed to `tenantStorage.run()` must be `async` for the context to propagate
@ -21,7 +26,9 @@ export const tenantStorage: AsyncLocalStorage<TenantContext> =
/** Returns the current tenant ID from async context, or undefined if none is set */
export function getTenantId(): string | undefined {
return tenantStorage.getStore()?.tenantId;
const tenantId = tenantStorage.getStore()?.tenantId;
if (isReservedTenantId(tenantId)) throw new TypeError('Reserved tenant identifier');
return tenantId;
}
/** Returns the current user ID from async context, or undefined if none is set */

View file

@ -29,6 +29,10 @@ export type {
} from './methods/response';
export {
createMethods,
createChannelRunStorage,
createChannelSummaryStorage,
CHANNEL_SUMMARY_MAX_PRIOR_ROWS,
ChannelRunConflict,
createSkillDeletionPersistenceAdapter,
AGENT_OWNER_CONTACT_RESOLVED_FIELD,
AgentSortCursorError,
@ -133,6 +137,7 @@ export {
runAsSystem,
scopedCacheKey,
SYSTEM_TENANT_ID,
isReservedTenantId,
} from './config/tenantContext';
export type { TenantContext } from './config/tenantContext';
export {

View file

@ -0,0 +1,41 @@
import { channelStorageConfigSchema } from 'librechat-data-provider';
import type { ChannelStorageConfig } from 'librechat-data-provider';
import type { ResponseOwner } from '~/models/response';
import { createChannelRunModel, createChannelLaneModel } from '~/models/channelRun';
/** Validate operation configuration before the caller closes irreversible admission fences. */
export function resolveChannelErasureConfig(
options: Partial<ChannelStorageConfig> = {},
): ChannelStorageConfig {
return channelStorageConfigSchema.parse(options);
}
/** The caller closes the response owner admission anchor before entering this cascade. */
export async function eraseChannelPersistenceForUser(
mongoose: typeof import('mongoose'),
owner: ResponseOwner,
options: Pick<Partial<ChannelStorageConfig>, 'erasureBatchSize'> = {},
): Promise<void> {
const { erasureBatchSize } = resolveChannelErasureConfig(options);
const tenantId = owner.tenantId || '_';
const runs = createChannelRunModel(mongoose);
const lanes = createChannelLaneModel(mongoose);
const filter = { tenantId, 'principal.userId': owner.userId };
// New lanes have an indexed owner. Older rows are recovered from the still-retained runs.
await lanes.deleteMany({ tenantId, userId: owner.userId });
const cursor = runs.find(filter).select('laneId').lean().cursor();
const batch = new Set<string>();
try {
for await (const run of cursor) {
batch.add(run.laneId);
if (batch.size === erasureBatchSize) {
await lanes.deleteMany({ tenantId, laneId: { $in: [...batch] } });
batch.clear();
}
}
if (batch.size) await lanes.deleteMany({ tenantId, laneId: { $in: [...batch] } });
} finally {
await cursor.close();
}
await runs.deleteMany(filter);
}

View file

@ -0,0 +1,37 @@
import { createChannelIndexGate } from './channelIndexes';
it('serializes required index builds and shares the readiness promise across concurrent admissions', async () => {
const order: string[] = [];
const first = {
modelName: 'First',
createIndexes: jest.fn(async () => {
order.push('first');
}),
};
const second = {
modelName: 'Second',
createIndexes: jest.fn(async () => {
order.push('second');
}),
};
const ready = createChannelIndexGate(() => [first, second]);
await Promise.all([ready(), ready(), ready()]);
await ready();
expect(order).toEqual(['first', 'second']);
expect(first.createIndexes).toHaveBeenCalledTimes(1);
expect(second.createIndexes).toHaveBeenCalledTimes(1);
});
it('fails closed on index failure and permits a later readiness retry', async () => {
const model = {
modelName: 'ChannelRun',
createIndexes: jest
.fn()
.mockRejectedValueOnce(new Error('index rejected'))
.mockResolvedValue(undefined),
};
const ready = createChannelIndexGate(() => [model]);
await expect(ready()).rejects.toThrow('index rejected');
await expect(ready()).resolves.toBeUndefined();
expect(model.createIndexes).toHaveBeenCalledTimes(2);
});

View file

@ -0,0 +1,19 @@
import { createIndexesWithRetry } from '~/utils/retry';
/** Unique indexes are correctness fences, even when automatic indexing is disabled. */
export function createChannelIndexGate(
models: () => Array<Parameters<typeof createIndexesWithRetry>[0]>,
afterReady?: () => Promise<void>,
): () => Promise<void> {
let pending: Promise<void> | undefined;
return () => {
pending ??= (async () => {
for (const model of models()) await createIndexesWithRetry(model);
await afterReady?.();
})().catch((error: unknown) => {
pending = undefined;
throw error;
});
return pending;
};
}

View file

@ -0,0 +1,29 @@
import type { ClientSession } from 'mongoose';
import type { ResponseOwner } from '~/models/response';
import { responseScopeKeys, initializeResponseScopes } from './responseScope';
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { createResponseScopeModel } from '~/models/responseScope';
/** All private channel persistence joins the same account-erasure admission fence. */
export async function prepareChannelOwnerAdmission(
mongoose: typeof import('mongoose'),
owner: ResponseOwner,
): Promise<(session: ClientSession) => Promise<boolean>> {
const current = getTenantId() || undefined;
if (
!owner.userId ||
owner.tenantId === SYSTEM_TENANT_ID ||
(current && current !== SYSTEM_TENANT_ID && current !== (owner.tenantId || undefined))
)
throw new TypeError('Channel owner does not match authenticated scope');
const [key] = responseScopeKeys(owner, '');
await initializeResponseScopes(mongoose, [key]);
return async (session) => {
const admitted = await createResponseScopeModel(mongoose).updateOne(
{ _id: key, closed: false },
{ $inc: { revision: 1 } },
{ session },
);
return admitted.matchedCount === 1;
};
}

View file

@ -0,0 +1,133 @@
import type { ChannelDeliveryOutcome, ChannelRunSnapshot } from 'librechat-data-provider';
import type { ChannelRunRecord } from '~/models/channelRun';
import { deliveryDisposition, snapshotPatch } from './channelProtocol';
function run(): ChannelRunRecord {
return {
tenantId: 'tenant',
channelId: 'channel',
agentId: 'agent',
laneId: 'lane',
runId: 'run',
principal: { userId: 'user', channelRevision: 'revision' },
payloadHash: 'hash',
sequence: 1,
admission: {
eventId: 'event',
externalChannelId: 'C',
externalThreadKey: 'C:thread',
externalMessageKey: 'C:event',
actor: { teamId: 'team', appId: 'app', externalUserId: 'actor' },
input: 'question',
},
receivedAt: 1,
state: 'delivering',
leaseEpoch: 1,
leaseExpiresAt: 1000,
cancellationRequested: false,
execution: {
requestId: 'run',
conversationId: 'conversation',
parentMessageId: 'parent',
responseId: 'response',
},
snapshot: { status: 'completed', responseId: 'response', text: 'answer', elapsedMs: 1 },
plan: {
revision: 'plan',
rendererVersion: 'test',
parts: [
{ operationId: 'required', kind: 'message', required: true, text: 'answer' },
{ operationId: 'optional', kind: 'message', required: false, text: 'footer' },
],
},
receipts: [
{
operationId: 'required',
outcome: 'confirmed',
attempts: 1,
externalMessageKey: 'C:required',
},
{
operationId: 'optional',
outcome: 'confirmed',
attempts: 1,
externalMessageKey: 'C:optional',
},
],
};
}
const outcomes: ChannelDeliveryOutcome[] = [
'planned',
'inflight',
'confirmed',
'retryable',
'unknown',
'permanent_failure',
];
const deliveryCases = outcomes.flatMap((required) =>
outcomes.map((optional) => ({ required, optional })),
);
it.each(deliveryCases)(
'finalizes required=$required and optional=$optional according to delivery policy',
({ required, optional }) => {
const current = run();
current.receipts[0].outcome = required;
current.receipts[1].outcome = optional;
const canFinish =
![required, optional].includes('inflight') &&
['confirmed', 'permanent_failure'].includes(required);
if (!canFinish) expect(() => deliveryDisposition(current, 100)).toThrow('delivery_incomplete');
else
expect(deliveryDisposition(current, 100)).toBe(
required === 'permanent_failure' ? 'failed' : 'delivered',
);
},
);
const statuses: ChannelRunSnapshot['status'][] = [
'running',
'requires_action',
'completed',
'failed',
'cancelled',
];
const snapshotCases = statuses.flatMap((previous) => statuses.map((next) => ({ previous, next })));
it.each(snapshotCases)(
'projects $previous to $next without reopening terminal work',
({ previous, next }) => {
const current = run();
delete current.plan;
current.snapshot = { ...current.snapshot!, status: previous };
const phases: Record<ChannelRunSnapshot['status'], ChannelRunRecord['state']> = {
running: 'executing',
requires_action: 'requires_action',
completed: 'delivering',
failed: 'delivering',
cancelled: 'delivering',
};
current.state = phases[previous];
const incoming = { ...current.snapshot, status: next };
if (['completed', 'failed', 'cancelled'].includes(previous) && previous !== next)
expect(() => snapshotPatch(current, incoming)).toThrow('invalid_transition');
else expect(snapshotPatch(current, incoming).state).toBe(phases[next]);
},
);
it('does not finalize while a progress operation is still active', () => {
const current = run();
current.progress = {
revision: 1,
rendererVersion: 'test',
text: 'progress',
outcome: 'inflight',
attempts: 1,
};
expect(() => deliveryDisposition(current, 100)).toThrow('delivery_incomplete');
});
it('rejects duplicate receipt identities instead of treating an incomplete plan as confirmed', () => {
const current = run();
current.receipts[1] = current.receipts[0];
expect(() => deliveryDisposition(current, 100)).toThrow('invalid_transition');
});

View file

@ -0,0 +1,240 @@
import { createHash } from 'node:crypto';
import type {
ChannelRunAdmission,
ChannelRunPrincipal,
ChannelRunSnapshot,
ChannelDeliveryPlan,
ChannelDeliveryReceipt,
} from 'librechat-data-provider';
import type { ChannelRunRecord } from '~/models/channelRun';
export interface ChannelLaneScope {
tenantId: string;
channelId: string;
agentId: string;
principal: ChannelRunPrincipal;
admission: Pick<ChannelRunAdmission, 'actor' | 'externalChannelId' | 'externalThreadKey'>;
}
export interface ChannelRunEnrollment extends ChannelLaneScope {
admission: ChannelRunAdmission;
}
export class ChannelRunConflict extends Error {
constructor(
readonly code: 'event_conflict' | 'stale_fence' | 'invalid_transition' | 'delivery_incomplete',
) {
super(code);
this.name = 'ChannelRunConflict';
}
}
export const digest = (value: object): string =>
createHash('sha256')
.update(
JSON.stringify(value, (_key, item: unknown) =>
item != null && typeof item === 'object' && !Array.isArray(item)
? Object.fromEntries(
Object.entries(item).sort(([a], [b]) => {
if (a === b) return 0;
return a < b ? -1 : 1;
}),
)
: item,
),
)
.digest('hex');
export function laneKey(scope: ChannelLaneScope): string {
return digest({
tenantId: scope.tenantId,
channelId: scope.channelId,
agentId: scope.agentId,
thread: scope.admission.externalThreadKey,
externalChannelId: scope.admission.externalChannelId,
actor: scope.admission.actor,
userId: scope.principal.userId,
revision: scope.principal.channelRevision,
linkRevision: scope.principal.linkRevision ?? null,
linkId: scope.principal.linkId ?? null,
});
}
export const activeStates: string[] = [
'queued',
'executing',
'requires_action',
'delivering',
'blocked',
];
/** A projection cannot grant a different response authority to the next actor-lane turn. */
export function assertSnapshotBinding(run: ChannelRunRecord, snapshot: ChannelRunSnapshot): void {
if (
(snapshot.status === 'completed' && !snapshot.responseId) ||
(snapshot.responseId != null && snapshot.responseId !== run.execution?.responseId)
)
throw new ChannelRunConflict('invalid_transition');
}
/** Confirm the provider identity appropriate to the immutable plan, on every confirmation path. */
export function assertDeliveryEvidence(
run: ChannelRunRecord,
receipt: ChannelDeliveryReceipt,
previous: ChannelDeliveryReceipt | undefined,
): void {
const part = run.plan?.parts.find((item) => item.operationId === receipt.operationId);
if (
!part ||
(part.kind === 'artifact' &&
(!part.artifactId ||
!run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))) ||
!previous ||
(previous.externalMessageKey != null &&
receipt.externalMessageKey != null &&
previous.externalMessageKey !== receipt.externalMessageKey) ||
(previous.externalFileId != null &&
receipt.externalFileId != null &&
previous.externalFileId !== receipt.externalFileId) ||
(receipt.upload != null && digest(receipt.upload) !== digest(previous.upload ?? {})) ||
(receipt.externalFileId != null &&
previous.upload != null &&
previous.upload.fileId !== receipt.externalFileId) ||
(receipt.outcome === 'confirmed' &&
((part.kind === 'message' && !receipt.externalMessageKey) ||
(part.kind === 'artifact' &&
(!receipt.externalFileId ||
(previous.upload != null && previous.upload.fileId !== receipt.externalFileId)))))
)
throw new ChannelRunConflict('invalid_transition');
}
export function leaseDeadline(now: number, leaseMs: number): number {
if (
!Number.isSafeInteger(now) ||
!Number.isSafeInteger(leaseMs) ||
leaseMs <= 0 ||
!Number.isSafeInteger(now + leaseMs)
)
throw new Error('Invalid channel lease');
return now + leaseMs;
}
/** Evidence is verified by the transport; storage binds it to the exact uncertain attempt. */
export function assertReconciliation(
previous: Omit<ChannelDeliveryReceipt, 'operationId'> | undefined,
receipt: Pick<ChannelDeliveryReceipt, 'attempts' | 'externalMessageKey' | 'externalFileId'>,
): void {
if (
!previous ||
!['unknown', 'confirmed'].includes(previous.outcome) ||
previous.attempts !== receipt.attempts ||
(!receipt.externalMessageKey && !receipt.externalFileId) ||
(previous.externalMessageKey && previous.externalMessageKey !== receipt.externalMessageKey) ||
(previous.externalFileId && previous.externalFileId !== receipt.externalFileId) ||
(previous.upload && previous.upload.fileId !== receipt.externalFileId) ||
(previous.outcome === 'confirmed' &&
(previous.externalMessageKey !== receipt.externalMessageKey ||
previous.externalFileId !== receipt.externalFileId))
)
throw new ChannelRunConflict('invalid_transition');
}
const terminalSnapshots = ['completed', 'failed', 'cancelled'];
export function snapshotPatch(
run: ChannelRunRecord,
snapshot: ChannelRunSnapshot,
): Pick<ChannelRunRecord, 'state' | 'snapshot'> {
assertSnapshotBinding(run, snapshot);
if (
run.snapshot &&
terminalSnapshots.includes(run.snapshot.status) &&
(run.snapshot.status !== snapshot.status ||
run.snapshot.responseId !== snapshot.responseId ||
run.snapshot.text !== snapshot.text)
)
throw new ChannelRunConflict('invalid_transition');
if (run.state === 'delivering' && !terminalSnapshots.includes(snapshot.status))
throw new ChannelRunConflict('invalid_transition');
let state: ChannelRunRecord['state'] = 'delivering';
if (snapshot.status === 'running') state = 'executing';
else if (snapshot.status === 'requires_action') state = 'requires_action';
const patch: Pick<ChannelRunRecord, 'state' | 'snapshot'> = {
state,
snapshot:
snapshot.summary == null && run.snapshot?.summary
? { ...snapshot, summary: run.snapshot.summary }
: snapshot,
};
if (run.plan && digest(patch.snapshot ?? {}) !== digest(run.snapshot ?? {}))
throw new ChannelRunConflict('invalid_transition');
return patch;
}
export function assertPlan(
run: ChannelRunRecord,
plan: ChannelDeliveryPlan,
maxParts: number,
): void {
if (
!run.snapshot ||
!terminalSnapshots.includes(run.snapshot.status) ||
run.state !== 'delivering' ||
!plan.parts.length ||
plan.parts.length > maxParts ||
new Set(plan.parts.map((part) => part.operationId)).size !== plan.parts.length ||
!plan.parts.some((part) => part.required) ||
plan.parts.some(
(part) =>
!part.operationId ||
!['message', 'artifact'].includes(part.kind) ||
(part.kind === 'artifact' &&
(!part.artifactId ||
!run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))),
)
)
throw new ChannelRunConflict('invalid_transition');
assertSnapshotBinding(run, run.snapshot);
}
export function hasPermanentDeliveryFailure(run: ChannelRunRecord): boolean {
return (
run.plan?.parts.some(
(part) =>
part.required &&
run.receipts.some(
(receipt) =>
receipt.operationId === part.operationId && receipt.outcome === 'permanent_failure',
),
) ?? false
);
}
/** A permanent required failure is terminal even though successful delivery is impossible.
* Unknown receipts never become eligible for resend. Optional unknown effects remain best effort. */
export function deliveryDisposition(
run: ChannelRunRecord,
maxParts: number,
): 'delivered' | 'failed' | 'cancelled' {
if (!run.plan || !run.snapshot) throw new ChannelRunConflict('delivery_incomplete');
assertPlan(run, run.plan, maxParts);
if (
run.receipts.length !== run.plan.parts.length ||
new Set(run.receipts.map((receipt) => receipt.operationId)).size !== run.receipts.length
)
throw new ChannelRunConflict('invalid_transition');
for (const receipt of run.receipts) assertDeliveryEvidence(run, receipt, receipt);
if (
run.receipts.some((receipt) => receipt.outcome === 'inflight') ||
run.progress?.outcome === 'inflight'
)
throw new ChannelRunConflict('delivery_incomplete');
const required = run.plan.parts
.filter((part) => part.required)
.map((part) => run.receipts.find((receipt) => receipt.operationId === part.operationId));
if (hasPermanentDeliveryFailure(run)) return 'failed';
if (required.some((receipt) => receipt?.outcome !== 'confirmed'))
throw new ChannelRunConflict('delivery_incomplete');
if (run.snapshot.status === 'completed') return 'delivered';
if (run.snapshot.status === 'cancelled') return 'cancelled';
return 'failed';
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,189 @@
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import type { ChannelRunRecord } from '~/models/channelRun';
import type { ChannelSummaryRow } from './channelSummary';
import { createChannelSummaryStorage, CHANNEL_SUMMARY_MAX_PRIOR_ROWS } from './channelSummary';
import { runAsSystem, tenantStorage } from '~/config/tenantContext';
import { createChannelRunModel } from '~/models/channelRun';
let mongo: MongoMemoryServer;
const storage = createChannelSummaryStorage(mongoose);
function record(sequence = 1): ChannelRunRecord {
return {
tenantId: 'tenant',
channelId: 'channel',
agentId: 'agent',
laneId: 'lane',
runId: `run-${sequence}`,
sequence,
state: 'delivered',
receivedAt: 1,
payloadHash: 'hash',
leaseEpoch: 1,
leaseExpiresAt: 0,
receipts: [],
cancellationRequested: false,
principal: {
userId: 'same-internal-user',
channelRevision: 'r1',
linkId: 'link',
linkRevision: 'l1',
},
admission: {
eventId: `event-${sequence}`,
externalChannelId: 'C1',
externalThreadKey: 'C1:123',
externalMessageKey: `C1:${sequence}`,
input: 'secret transcript',
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'ACTOR-A' },
},
snapshot: {
status: 'completed',
text: 'secret answer',
elapsedMs: 10,
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
accounting: {
status: 'available',
currency: 'USD',
amount: '0.1',
basis: 'application_billing',
coverage: 'complete',
reason: 'priced',
calls: [],
},
},
};
}
async function read(scope: ChannelRunRecord): Promise<ChannelSummaryRow[]> {
return tenantStorage.run({ tenantId: scope.tenantId }, async () => {
const rows: ChannelSummaryRow[] = [];
for await (const row of storage.priorMetrics(scope)) rows.push(row);
return rows;
});
}
beforeAll(async () => {
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
await mongoose.connect(mongo.getUri());
createChannelRunModel(mongoose);
await mongoose.models.ChannelRun.init();
}, 60000);
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
beforeEach(async () => {
await runAsSystem(async () => mongoose.models.ChannelRun.deleteMany({}));
});
it('reads newest prior metrics only, using the existing lane index, without transcripts', async () => {
await runAsSystem(async () =>
mongoose.models.ChannelRun.insertMany([record(1), record(2), record(3)]),
);
const rows = await read(record(3));
expect(rows.map((row) => row.sequence)).toEqual([2, 1]);
expect(rows[0]).toEqual({
sequence: 2,
snapshot: {
status: 'completed',
elapsedMs: 10,
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
accounting: {
status: 'available',
currency: 'USD',
amount: '0.1',
basis: 'application_billing',
coverage: 'complete',
reason: 'priced',
calls: [],
},
},
});
expect(JSON.stringify(rows)).not.toContain('secret');
expect(JSON.stringify(rows)).not.toContain('same-internal-user');
const indexes = await mongoose.models.ChannelRun.collection.indexes();
expect(
indexes.some(
(index) =>
JSON.stringify(index.key) === JSON.stringify({ tenantId: 1, laneId: 1, sequence: 1 }),
),
).toBe(true);
});
it.each([
'tenant',
'lane',
'channel',
'agent',
'user',
'channelRevision',
'link',
'linkRevision',
'team',
'app',
'actor',
'thread',
'externalChannel',
])('excludes a foreign %s even when a malformed row reuses the same lane', async (kind) => {
const foreign = record(1);
if (kind === 'tenant') foreign.tenantId = 'other';
if (kind === 'lane') foreign.laneId = 'other';
if (kind === 'channel') foreign.channelId = 'other';
if (kind === 'agent') foreign.agentId = 'other';
if (kind === 'user') foreign.principal.userId = 'other';
if (kind === 'channelRevision') foreign.principal.channelRevision = 'other';
if (kind === 'link') foreign.principal.linkId = 'other';
if (kind === 'linkRevision') foreign.principal.linkRevision = 'other';
if (kind === 'team') foreign.admission.actor.teamId = 'other';
if (kind === 'app') foreign.admission.actor.appId = 'other';
if (kind === 'actor') foreign.admission.actor.externalUserId = 'ACTOR-B';
if (kind === 'thread') foreign.admission.externalThreadKey = 'other';
if (kind === 'externalChannel') foreign.admission.externalChannelId = 'other';
await runAsSystem(async () => mongoose.models.ChannelRun.create(foreign));
expect(await read(record(2))).toEqual([]);
});
it('keeps failed/cancelled/legacy attempts visible as metric coverage gaps', async () => {
await runAsSystem(async () =>
mongoose.models.ChannelRun.insertMany([
{ ...record(1), state: 'failed', snapshot: undefined },
{ ...record(2), state: 'cancelled' },
{ ...record(3), state: 'blocked' },
]),
);
expect((await read(record(4))).map((row) => row.sequence)).toEqual([3, 2, 1]);
});
it('bounds long-history reads at newest 1000 rows without silently altering the sequence cutoff', async () => {
const count = CHANNEL_SUMMARY_MAX_PRIOR_ROWS + 3;
await runAsSystem(async () =>
mongoose.models.ChannelRun.insertMany(Array.from({ length: count }, (_, i) => record(i + 1))),
);
const rows = await read(record(count));
expect(rows).toHaveLength(CHANNEL_SUMMARY_MAX_PRIOR_ROWS);
expect(rows[0].sequence).toBe(count - 1);
expect(rows[rows.length - 1]?.sequence).toBe(count - CHANNEL_SUMMARY_MAX_PRIOR_ROWS);
});
it('throws for invalid sequence and database failure instead of disguising outages as absence', async () => {
await expect(read(record(0))).rejects.toThrow('Invalid channel summary sequence');
const other = new mongoose.Mongoose();
await expect(
(async () => {
for await (const row of createChannelSummaryStorage(other).priorMetrics(record(2))) void row;
})(),
).rejects.toThrow();
});
it('uses the configured history work bound while retaining the newest prior rows', async () => {
await runAsSystem(async () =>
mongoose.models.ChannelRun.insertMany(Array.from({ length: 5 }, (_, i) => record(i + 1))),
);
const configured = createChannelSummaryStorage(mongoose, { maxPriorSummaryRows: 2 });
const rows = await tenantStorage.run({ tenantId: 'tenant' }, async () => {
const result: ChannelSummaryRow[] = [];
for await (const row of configured.priorMetrics({ ...record(6), sequence: 6 }))
result.push(row);
return result;
});
expect(rows.map((row) => row.sequence)).toEqual([5, 4]);
});

View file

@ -0,0 +1,86 @@
import { channelStorageConfigSchema } from 'librechat-data-provider';
import type {
ChannelRunSnapshot,
ResponseAccounting,
ChannelStorageConfig,
} from 'librechat-data-provider';
import type { ChannelRunRecord } from '~/models/channelRun';
export type ChannelSummaryScope = Pick<
ChannelRunRecord,
'tenantId' | 'channelId' | 'agentId' | 'laneId' | 'principal' | 'sequence' | 'admission'
>;
export interface ChannelSummaryRow {
sequence: number;
snapshot?: Pick<ChannelRunSnapshot, 'status' | 'elapsedMs' | 'usage'> & {
accounting?: Pick<
ResponseAccounting,
'status' | 'currency' | 'coverage' | 'amount' | 'basis' | 'reason'
> & { calls?: Array<Pick<ResponseAccounting['calls'][number], 'reason'>> };
};
}
export interface ChannelSummaryStorage {
/** Newest prior attempts, with a fixed presentation safety bound; not a billing ledger. */
priorMetrics(scope: ChannelSummaryScope): AsyncIterable<ChannelSummaryRow>;
}
/** Bounds footer work independently of thread length. Older rows become explicit coverage gaps. */
export const CHANNEL_SUMMARY_MAX_PRIOR_ROWS: number = channelStorageConfigSchema.parse(
{},
).maxPriorSummaryRows;
export function createChannelSummaryStorage(
mongoose: typeof import('mongoose'),
options: Pick<Partial<ChannelStorageConfig>, 'maxPriorSummaryRows'> = {},
): ChannelSummaryStorage {
const { maxPriorSummaryRows } = channelStorageConfigSchema.parse(options);
return {
async *priorMetrics(scope) {
if (!Number.isSafeInteger(scope.sequence) || scope.sequence < 1)
throw new Error('Invalid channel summary sequence');
const cursor = mongoose
.model<ChannelRunRecord>('ChannelRun')
.find({
tenantId: scope.tenantId,
laneId: scope.laneId,
sequence: { $gte: 1, $lt: scope.sequence },
channelId: scope.channelId,
agentId: scope.agentId,
'principal.userId': scope.principal.userId,
'principal.channelRevision': scope.principal.channelRevision,
'principal.linkId': scope.principal.linkId ?? null,
'principal.linkRevision': scope.principal.linkRevision ?? null,
'admission.actor.teamId': scope.admission.actor.teamId,
'admission.actor.appId': scope.admission.actor.appId,
'admission.actor.externalUserId': scope.admission.actor.externalUserId,
'admission.externalChannelId': scope.admission.externalChannelId,
'admission.externalThreadKey': scope.admission.externalThreadKey,
})
.select({
_id: 0,
sequence: 1,
'snapshot.status': 1,
'snapshot.elapsedMs': 1,
'snapshot.usage': 1,
'snapshot.accounting.status': 1,
'snapshot.accounting.currency': 1,
'snapshot.accounting.coverage': 1,
'snapshot.accounting.amount': 1,
'snapshot.accounting.basis': 1,
'snapshot.accounting.reason': 1,
'snapshot.accounting.calls.reason': 1,
})
.sort({ sequence: -1 })
.limit(maxPriorSummaryRows)
.lean<ChannelSummaryRow[]>()
.cursor();
try {
for await (const row of cursor) yield row;
} finally {
await cursor.close();
}
},
};
}

View file

@ -5,6 +5,7 @@ export type {
PermissionPrincipalRecord,
PermissionPrincipalData,
} from './provisioning';
import type { ChannelStorageConfig } from 'librechat-data-provider';
import { createSubagentContentMethods, type SubagentContentMethods } from './subagentContent';
export type {
SubagentContentScope,
@ -68,6 +69,20 @@ import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCa
/* Agent API Keys */
import { createAgentApiKeyMethods, type AgentApiKeyMethods } from './agentApiKey';
export { createChannelRunStorage, ChannelRunConflict } from './channelRun';
export { createChannelSummaryStorage, CHANNEL_SUMMARY_MAX_PRIOR_ROWS } from './channelSummary';
export type {
ChannelSummaryStorage,
ChannelSummaryScope,
ChannelSummaryRow,
} from './channelSummary';
export type {
ChannelRunStorage,
ChannelRunEnrollment,
ChannelLaneScope,
ChannelRunRecord,
ChannelLaneRecord,
} from './channelRun';
/* MCP Servers */
import { createMCPServerMethods, type MCPServerMethods } from './mcpServer';
import { createCodeEnvironmentMethods, type CodeEnvironmentMethods } from './codeEnvironment';
@ -341,6 +356,7 @@ export type AllMethods = PermissionPrincipalStorage &
/** Dependencies injected from the api layer into createMethods */
export interface CreateMethodsDeps {
getChannelStorageConfig?: () => Promise<Partial<ChannelStorageConfig>>;
/** Matches a model name to a canonical key. From @librechat/api. */
matchModelName?: (model: string, endpoint?: string) => string | undefined;
/** Finds the first key in values whose key is a substring of model. From @librechat/api. */
@ -529,7 +545,10 @@ export function createMethods(
};
const agentMethods = createAgentMethods(mongoose, agentDeps);
return {
...createUserMethods(mongoose, { getCache: deps.getCache }),
...createUserMethods(mongoose, {
getCache: deps.getCache,
getChannelStorageConfig: deps.getChannelStorageConfig,
}),
...createToolApprovalGrantMethods(mongoose),
...createSessionMethods(mongoose),
...createTokenMethods(mongoose),

View file

@ -1,6 +1,7 @@
import mongoose, { FilterQuery } from 'mongoose';
import { AUTH_USER_DOC_MAX_AGE_MS } from 'librechat-data-provider';
import type {
ChannelStorageConfig,
BalanceRefillMode,
RefillIntervalUnit,
StatefulCodeEnvironment,
@ -16,6 +17,10 @@ import type {
} from '~/types';
import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types';
import type { CacheStore } from '~/types';
import {
eraseChannelPersistenceForUser,
resolveChannelErasureConfig,
} from '~/methods/channelErasure';
import { closeResponseScopes, eraseResponseRecords } from './responseScope';
import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search';
import { evictAuthUserDocsForUsers } from '~/utils/eviction';
@ -35,6 +40,7 @@ const TWO_FACTOR_ENROLLMENT_PROJECTION =
'+totpSecret +backupCodes +pendingTotpSecret +pendingBackupCodes +twoFactorAcknowledgementNonceHash +twoFactorFinalizationNonceHash';
interface UserMethodDeps {
getChannelStorageConfig?: () => Promise<Partial<ChannelStorageConfig>>;
getCache?: (key: string) => CacheStore | undefined;
/** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */
delay?: (ms: number) => Promise<void>;
@ -644,8 +650,18 @@ export function createUserMethods(
const User = mongoose.models.User;
const owner = await User.findById(userId).select('tenantId').lean<Pick<IUser, 'tenantId'>>();
if (owner) {
const channelConfig = resolveChannelErasureConfig(
(await deps.getChannelStorageConfig?.()) ?? {},
);
await closeResponseScopes(mongoose, { userId, tenantId: owner.tenantId });
await eraseResponseRecords(mongoose, { userId, tenantId: owner.tenantId });
// Admission writes the same owner anchor in its insertion transaction.
// Closing it therefore precedes both this sweep and every delayed retry.
await eraseChannelPersistenceForUser(
mongoose,
{ userId, tenantId: owner.tenantId },
channelConfig,
);
}
await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId });
const result = await User.deleteOne({ _id: userId });

View file

@ -0,0 +1,179 @@
import { Schema } from 'mongoose';
import type {
ChannelRunView,
ChannelRunPrincipal,
ChannelDeliveryReceipt,
} from 'librechat-data-provider';
import type { Model } from 'mongoose';
import { applyTenantIsolation } from './plugins/tenantIsolation';
export interface ChannelRunRecord extends Omit<ChannelRunView, 'revision'> {
tenantId: string;
agentId: string;
laneId: string;
principal: ChannelRunPrincipal;
payloadHash: string;
receivedAt: number;
leaseOwner?: string;
leaseEpoch: number;
leaseExpiresAt: number;
/** Durable fairness stamp; not execution authority or a retry policy. */
claimCheckedAt?: number;
/** Persisted before dispatch; every retry attaches to this exact identity. */
execution?: {
protocol?: 'responses';
responseId?: string;
apiKeyId?: string;
encryptedApiKey?: string;
requestId: string;
conversationId: string;
parentMessageId: string;
generationCreatedAt?: number;
deliveryKey?: string;
enqueueAttemptedAt?: number;
};
errorCode?: string;
deliveredAt?: number;
}
export interface ChannelLaneRecord {
tenantId: string;
userId: string;
channelId: string;
laneId: string;
nextSequence: number;
activeRunId?: string;
conversationId?: string;
responseId?: string;
}
export function createChannelRunModel(
mongoose: typeof import('mongoose'),
): Model<ChannelRunRecord> {
const receipt = new Schema<ChannelDeliveryReceipt>(
{
operationId: { type: String, required: true },
outcome: {
type: String,
required: true,
enum: ['planned', 'inflight', 'confirmed', 'retryable', 'unknown', 'permanent_failure'],
},
attempts: { type: Number, required: true },
externalMessageKey: String,
externalFileId: String,
retryAt: Number,
errorCode: String,
upload: {
type: new Schema(
{
fileId: { type: String, required: true },
stage: {
type: String,
required: true,
enum: ['allocated', 'uploaded'],
},
},
{ _id: false },
),
},
},
{ _id: false },
);
const schema = new Schema<ChannelRunRecord>(
{
tenantId: { type: String, required: true },
channelId: { type: String, required: true },
agentId: { type: String, required: true },
laneId: { type: String, required: true },
runId: { type: String, required: true },
state: {
type: String,
required: true,
enum: [
'queued',
'executing',
'requires_action',
'delivering',
'delivered',
'failed',
'cancelled',
'blocked',
],
},
admission: { type: Schema.Types.Mixed, required: true },
principal: { type: Schema.Types.Mixed, required: true },
payloadHash: { type: String, required: true },
sequence: { type: Number, required: true },
receivedAt: { type: Number, required: true },
leaseOwner: String,
leaseEpoch: { type: Number, required: true, default: 0 },
leaseExpiresAt: { type: Number, required: true, default: 0 },
claimCheckedAt: { type: Number, default: 0 },
execution: Schema.Types.Mixed,
snapshot: Schema.Types.Mixed,
progress: Schema.Types.Mixed,
plan: Schema.Types.Mixed,
receipts: { type: [receipt], required: true, default: [] },
cancellationRequested: { type: Boolean, required: true, default: false },
errorCode: String,
deliveredAt: Number,
},
{ timestamps: true, minimize: false },
);
schema.index({ tenantId: 1, runId: 1 }, { unique: true });
schema.index({ tenantId: 1, 'principal.userId': 1 });
schema.index(
{
tenantId: 1,
channelId: 1,
'admission.actor.teamId': 1,
'admission.actor.appId': 1,
'admission.eventId': 1,
},
{ unique: true, name: 'channel_run_event_v2' },
);
schema.index({ tenantId: 1, laneId: 1, sequence: 1 }, { unique: true });
schema.index(
{ tenantId: 1, laneId: 1, state: 1, sequence: 1 },
{ name: 'channel_run_active_predecessor' },
);
schema.index({ tenantId: 1, channelId: 1, state: 1, leaseExpiresAt: 1, receivedAt: 1 });
schema.index(
{
tenantId: 1,
channelId: 1,
state: 1,
claimCheckedAt: 1,
receivedAt: 1,
sequence: 1,
leaseExpiresAt: 1,
},
{ name: 'channel_run_fair_claim' },
);
schema.index({ state: 1, _id: 1 });
schema.index({ tenantId: 1, channelId: 1, 'receipts.externalMessageKey': 1 });
applyTenantIsolation(schema);
return mongoose.models.ChannelRun ?? mongoose.model<ChannelRunRecord>('ChannelRun', schema);
}
export function createChannelLaneModel(
mongoose: typeof import('mongoose'),
): Model<ChannelLaneRecord> {
const schema = new Schema<ChannelLaneRecord>(
{
tenantId: { type: String, required: true },
userId: { type: String, required: true },
channelId: { type: String, required: true },
laneId: { type: String, required: true },
nextSequence: { type: Number, required: true, default: 0 },
activeRunId: String,
conversationId: String,
responseId: String,
},
{ timestamps: true },
);
schema.index({ tenantId: 1, laneId: 1 }, { unique: true });
schema.index({ tenantId: 1, userId: 1 });
applyTenantIsolation(schema);
return mongoose.models.ChannelLane ?? mongoose.model<ChannelLaneRecord>('ChannelLane', schema);
}

View file

@ -1,5 +1,6 @@
import { createAgentQueuedTurnModel, createAgentQueuedTurnSequenceModel } from './queuedTurn';
import { createChannelRunModel, createChannelLaneModel } from './channelRun';
import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';
import { createScheduleModel, createScheduleRunModel } from './schedule';
import { getTenantIndexMigrationHint } from '~/migrations/tenantIndexes';
@ -75,6 +76,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
SubagentContent: ReturnType<typeof createSubagentContentModel>;
Agent: ReturnType<typeof createAgentModel>;
AgentApiKey: ReturnType<typeof createAgentApiKeyModel>;
ChannelRun: ReturnType<typeof createChannelRunModel>;
ChannelLane: ReturnType<typeof createChannelLaneModel>;
AgentCategory: ReturnType<typeof createAgentCategoryModel>;
MCPServer: ReturnType<typeof createMCPServerModel>;
Role: ReturnType<typeof createRoleModel>;
@ -132,6 +135,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
SubagentContent: createSubagentContentModel(mongoose),
Agent: createAgentModel(mongoose),
AgentApiKey: createAgentApiKeyModel(mongoose),
ChannelRun: createChannelRunModel(mongoose),
ChannelLane: createChannelLaneModel(mongoose),
AgentCategory: createAgentCategoryModel(mongoose),
MCPServer: createMCPServerModel(mongoose),
Role: createRoleModel(mongoose),

View file

@ -1,6 +1,7 @@
import { Schema } from 'mongoose';
import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider';
import { applySearchTokens } from '~/models/plugins/searchTokens';
import { isReservedTenantId } from '~/config/tenantContext';
import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search';
import { IUser } from '~/types';
@ -232,6 +233,10 @@ const userSchema: Schema<IUser> = new Schema<IUser>(
tenantId: {
type: String,
index: true,
validate: {
validator: (value: string | undefined) => !isReservedTenantId(value),
message: 'Reserved tenant identifier',
},
},
},
{ timestamps: true },