mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
📬 feat: Persist Channel Events and Delivery Receipts (#17595)
* 📬 feat: Persist Channel Events and Delivery Receipts
* fix: enforce channel run identity and fenced recovery
* fix: fence and erase channel state during account deletion
* refactor: share channel owner admission with private persistence
* refactor: name channel account-erasure boundary explicitly
* fix: prove channel identity and delivery authority at storage boundaries
* refactor: enforce channel delivery protocol and terminal lifecycle
* fix: enforce channel tenant and operation boundaries
* fix: initialize channel config from its composition owner
* test: verify config installs channel storage reader
* test: complete config initializer contract in model mocks
* test: include channel config reader in migration fixture
* test: await SSE subscription before advancing keepalive time
* test: Observe and Drain Concurrent Fixture Work
This commit is contained in:
parent
1458e841e4
commit
7dd4fafdf3
48 changed files with 3816 additions and 54 deletions
|
|
@ -5,16 +5,19 @@ const {
|
|||
findMatchingPattern,
|
||||
isDeploymentSkillId,
|
||||
createMessageBudgetReader,
|
||||
createChannelStorageConfigReader,
|
||||
} = require('@librechat/api');
|
||||
const getLogStores = require('~/cache/getLogStores');
|
||||
|
||||
const messageBudget = createMessageBudgetReader();
|
||||
const channelStorageConfig = createChannelStorageConfigReader();
|
||||
|
||||
const methods = createMethods(mongoose, {
|
||||
matchModelName,
|
||||
findMatchingPattern,
|
||||
isExternalSkillId: isDeploymentSkillId,
|
||||
getCache: getLogStores,
|
||||
getChannelStorageConfig: channelStorageConfig.getConfig,
|
||||
getMCPAppMessageBudget: messageBudget.getBudget,
|
||||
});
|
||||
|
||||
|
|
@ -28,5 +31,6 @@ const seedDatabase = async () => {
|
|||
module.exports = {
|
||||
...methods,
|
||||
initializeMessageBudget: messageBudget.initialize,
|
||||
initializeChannelStorageConfig: channelStorageConfig.initialize,
|
||||
seedDatabase,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,15 +3,21 @@ jest.mock('@librechat/api', () => ({
|
|||
matchModelName: jest.fn(),
|
||||
findMatchingPattern: jest.fn(),
|
||||
isDeploymentSkillId: jest.fn(),
|
||||
createChannelStorageConfigReader: jest.fn(() => ({
|
||||
getConfig: jest.fn(),
|
||||
initialize: jest.fn(),
|
||||
})),
|
||||
createMessageBudgetReader: jest.fn(() => ({ getBudget: jest.fn(), initialize: jest.fn() })),
|
||||
}));
|
||||
jest.mock('~/cache/getLogStores', () => jest.fn());
|
||||
|
||||
const { createMethods } = require('@librechat/data-schemas');
|
||||
const { createMessageBudgetReader } = require('@librechat/api');
|
||||
const { initializeMessageBudget } = require('./index');
|
||||
const { createMessageBudgetReader, createChannelStorageConfigReader } = require('@librechat/api');
|
||||
const { initializeMessageBudget, initializeChannelStorageConfig } = require('./index');
|
||||
const reader = createMessageBudgetReader.mock.results[0].value;
|
||||
const { getMCPAppMessageBudget } = createMethods.mock.calls[0][1];
|
||||
const channelStorageConfigReader = createChannelStorageConfigReader.mock.results[0].value;
|
||||
const { getMCPAppMessageBudget, getChannelStorageConfig: channelStorageConfigDependency } =
|
||||
createMethods.mock.calls[0][1];
|
||||
|
||||
describe('message App budget dependency wiring', () => {
|
||||
it('passes the same reader to storage and initializer to the composition root', () => {
|
||||
|
|
@ -20,3 +26,11 @@ describe('message App budget dependency wiring', () => {
|
|||
expect(reader.getBudget).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('channel storage configuration dependency wiring', () => {
|
||||
it('injects the deployment reader without reading config during module initialization', () => {
|
||||
expect(channelStorageConfigDependency).toBe(channelStorageConfigReader.getConfig);
|
||||
expect(initializeChannelStorageConfig).toBe(channelStorageConfigReader.initialize);
|
||||
expect(channelStorageConfigReader.getConfig).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ jest.mock('@librechat/api', () => ({
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
saveMessage: jest.fn(async (req, message) => message),
|
||||
getConvo: jest.fn(async () => null),
|
||||
getMessages: jest.fn(async () => []),
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ const mockGetCacheMultiplier = jest.fn(() => 1);
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
getMultiplier: (...args) => mockGetMultiplier(...args),
|
||||
getCacheMultiplier: (...args) => mockGetCacheMultiplier(...args),
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const mockSpendTokens = jest.fn().mockResolvedValue(undefined);
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
spendTokens: mockSpendTokens,
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ jest.mock('@librechat/api', () => ({
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
saveMessage: jest.fn(async (req, message) => message),
|
||||
getConvo: jest.fn(async () => null),
|
||||
getMessages: jest.fn(async () => []),
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ const mockRecordCollectedUsage = jest
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
spendTokens: (...args) => mockSpendTokens(...args),
|
||||
spendStructuredTokens: (...args) => mockSpendStructuredTokens(...args),
|
||||
getMultiplier: mockGetMultiplier,
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ jest.mock('@librechat/api', () => ({
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
saveMessage: (...args) => mockSaveMessage(...args),
|
||||
getPersistedPrivateTextId: async (...args) =>
|
||||
(await mockHasPersistedPrivateText(...args)) ? 'protected-row-id' : null,
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ jest.mock('@librechat/api', () => ({
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
saveMessage: (...args) => mockSaveMessage(...args),
|
||||
}));
|
||||
|
||||
|
|
@ -86,6 +87,28 @@ function mockSubscribeSuccess() {
|
|||
});
|
||||
}
|
||||
|
||||
/** A real HTTP socket must reach subscribe before fake time can exercise its keepalive. */
|
||||
function openSilentStream() {
|
||||
let resolveAttached;
|
||||
const subscribed = new Promise((resolve) => {
|
||||
resolveAttached = resolve;
|
||||
});
|
||||
mockGenerationJobManager.subscribe.mockImplementation(async (_streamId, _writeEvent, onDone) => {
|
||||
resolveAttached(onDone);
|
||||
return { unsubscribe: jest.fn() };
|
||||
});
|
||||
const pending = request(app)
|
||||
.get('/agents/chat/stream/stream-123')
|
||||
.then((res) => res);
|
||||
const attached = Promise.race([
|
||||
subscribed,
|
||||
pending.then((res) => {
|
||||
throw new Error(`Stream ended before subscribing: HTTP ${res.status}`);
|
||||
}),
|
||||
]);
|
||||
return { pending, attached };
|
||||
}
|
||||
|
||||
describe('SSE stream tenant isolation', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
|
@ -411,19 +434,8 @@ describe('SSE stream tenant isolation', () => {
|
|||
status: 'running',
|
||||
createdAt: 1000,
|
||||
});
|
||||
let finish;
|
||||
mockGenerationJobManager.subscribe.mockImplementation(
|
||||
async (_streamId, _writeEvent, onDone) => {
|
||||
finish = onDone;
|
||||
return { unsubscribe: jest.fn() };
|
||||
},
|
||||
);
|
||||
|
||||
const pending = request(app)
|
||||
.get('/agents/chat/stream/stream-123')
|
||||
.then((res) => res);
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const { pending, attached } = openSilentStream();
|
||||
const finish = await attached;
|
||||
await jest.advanceTimersByTimeAsync(60_000);
|
||||
finish({ final: true });
|
||||
const res = await pending;
|
||||
|
|
@ -470,19 +482,8 @@ describe('SSE stream tenant isolation', () => {
|
|||
status: 'running',
|
||||
createdAt: 1000,
|
||||
});
|
||||
let finish;
|
||||
mockGenerationJobManager.subscribe.mockImplementation(
|
||||
async (_streamId, _writeEvent, onDone) => {
|
||||
finish = onDone;
|
||||
return { unsubscribe: jest.fn() };
|
||||
},
|
||||
);
|
||||
|
||||
const pending = request(app)
|
||||
.get('/agents/chat/stream/stream-123')
|
||||
.then((res) => res);
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const { pending, attached } = openSilentStream();
|
||||
const finish = await attached;
|
||||
await jest.advanceTimersByTimeAsync(60_000);
|
||||
finish({ final: true });
|
||||
const res = await pending;
|
||||
|
|
@ -502,19 +503,8 @@ describe('SSE stream tenant isolation', () => {
|
|||
status: 'running',
|
||||
createdAt: 1000,
|
||||
});
|
||||
let finish;
|
||||
mockGenerationJobManager.subscribe.mockImplementation(
|
||||
async (_streamId, _writeEvent, onDone) => {
|
||||
finish = onDone;
|
||||
return { unsubscribe: jest.fn() };
|
||||
},
|
||||
);
|
||||
|
||||
const pending = request(app)
|
||||
.get('/agents/chat/stream/stream-123')
|
||||
.then((res) => res);
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const { pending, attached } = openSilentStream();
|
||||
const finish = await attached;
|
||||
await jest.advanceTimersByTimeAsync(60_000);
|
||||
|
||||
expect(mockGenerationJobManager.subscribe).toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ jest.mock('~/server/services/start/tools', () => ({
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
getApplicableConfigs: jest.fn().mockResolvedValue([]),
|
||||
getUserPrincipals: jest.fn().mockResolvedValue([]),
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ jest.mock('@librechat/data-schemas', () => {
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
getApplicableConfigs: jest.fn().mockResolvedValue([]),
|
||||
getUserPrincipals: jest.fn().mockResolvedValue([]),
|
||||
}));
|
||||
|
|
@ -61,14 +62,16 @@ jest.mock('@librechat/api', () => ({
|
|||
// ── Tests ──────────────────────────────────────────────────────────────
|
||||
|
||||
const { getAppConfig, invalidateConfigCaches } = require('../app');
|
||||
const { initializeMessageBudget } = require('~/models');
|
||||
const { initializeMessageBudget, initializeChannelStorageConfig } = require('~/models');
|
||||
// Capture module-composition calls before clearMocks resets the spy history.
|
||||
const budgetInitializations = [...initializeMessageBudget.mock.calls];
|
||||
const channelInitializations = [...initializeChannelStorageConfig.mock.calls];
|
||||
const invalidationOptions = mockCreateConfigCacheInvalidation.mock.calls[0][0];
|
||||
const pubSubOptions = mockCreateRedisPubSub.mock.calls[0][0];
|
||||
|
||||
test('Config supplies its actual reader exactly once during composition', () => {
|
||||
expect(budgetInitializations).toEqual([[getAppConfig]]);
|
||||
expect(channelInitializations).toEqual([[getAppConfig]]);
|
||||
});
|
||||
|
||||
describe('invalidateConfigCaches', () => {
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ const {
|
|||
|
||||
// Config owns the reader; models never import this module to obtain it.
|
||||
db.initializeMessageBudget(getAppConfig);
|
||||
db.initializeChannelStorageConfig(getAppConfig);
|
||||
|
||||
/**
|
||||
* Admin config writes clear the per-principal override caches, the MCP config-source servers
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ jest.mock('@librechat/api', () => ({
|
|||
fetchModels: jest.fn(),
|
||||
}));
|
||||
jest.mock('./app');
|
||||
jest.mock('~/models', () => ({ getUserKeyValues: jest.fn(), initializeMessageBudget: jest.fn() }));
|
||||
jest.mock('~/models', () => ({
|
||||
getUserKeyValues: jest.fn(),
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
}));
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
...jest.requireActual('@librechat/data-schemas'),
|
||||
logger: { debug: jest.fn(), error: jest.fn(), warn: jest.fn(), info: jest.fn() },
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ jest.mock('@librechat/data-schemas', () => ({
|
|||
jest.mock('~/models', () => ({
|
||||
getUserKeyValues: jest.fn(),
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
}));
|
||||
|
||||
const exampleConfig = {
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ const { ImportBatchBuilder } = require('./importBatchBuilder');
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
bulkIncrementTagCounts: jest.fn(),
|
||||
bulkSaveConvos: jest.fn(),
|
||||
bulkSaveMessages: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ const importConversations = require('./importConversations');
|
|||
|
||||
jest.mock('~/models', () => ({
|
||||
initializeMessageBudget: jest.fn(),
|
||||
initializeChannelStorageConfig: jest.fn(),
|
||||
bulkIncrementTagCounts: jest.fn(),
|
||||
bulkSaveConvos: jest.fn(),
|
||||
bulkSaveMessages: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
jest.mock('../connect', () => jest.fn().mockResolvedValue(true));
|
||||
jest.mock('@librechat/api', () => ({
|
||||
ensureRequiredCollectionsExist: jest.fn().mockResolvedValue(undefined),
|
||||
createChannelStorageConfigReader: jest.fn(() => ({
|
||||
initialize: jest.fn(),
|
||||
getConfig: jest.fn(),
|
||||
})),
|
||||
createMessageBudgetReader: jest.fn(() => ({ initialize: jest.fn(), getBudget: jest.fn() })),
|
||||
matchModelName: jest.fn(),
|
||||
findMatchingPattern: jest.fn(),
|
||||
|
|
|
|||
35
packages/api/src/channels/config.spec.ts
Normal file
35
packages/api/src/channels/config.spec.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import { createChannelStorageConfigReader } from './config';
|
||||
|
||||
it('fails closed until the configuration service installs its reader', async () => {
|
||||
await expect(createChannelStorageConfigReader().getConfig()).rejects.toThrow(
|
||||
'not been initialized',
|
||||
);
|
||||
});
|
||||
|
||||
it('reads live base deployment bounds at each operation boundary', async () => {
|
||||
const config = { endpoints: { agents: { channels: { erasureBatchSize: 2 } } } };
|
||||
const read = jest.fn(async () => config as AppConfig);
|
||||
const effective = createChannelStorageConfigReader();
|
||||
effective.initialize(read);
|
||||
expect(await effective.getConfig()).toMatchObject({ erasureBatchSize: 2, claimBatchSize: 25 });
|
||||
config.endpoints.agents.channels.erasureBatchSize = 3;
|
||||
expect((await effective.getConfig()).erasureBatchSize).toBe(3);
|
||||
expect(read).toHaveBeenCalledTimes(2);
|
||||
expect(read).toHaveBeenLastCalledWith({ baseOnly: true });
|
||||
effective.initialize(async () => ({}) as AppConfig);
|
||||
expect((await effective.getConfig()).erasureBatchSize).toBe(250);
|
||||
});
|
||||
|
||||
it('rejects invalid bounds and propagates config outages', async () => {
|
||||
const effective = createChannelStorageConfigReader();
|
||||
effective.initialize(
|
||||
async () => ({ endpoints: { agents: { channels: { erasureBatchSize: 0 } } } }) as AppConfig,
|
||||
);
|
||||
await expect(effective.getConfig()).rejects.toThrow();
|
||||
const failure = new Error('config unavailable');
|
||||
effective.initialize(async () => {
|
||||
throw failure;
|
||||
});
|
||||
await expect(effective.getConfig()).rejects.toBe(failure);
|
||||
});
|
||||
22
packages/api/src/channels/config.ts
Normal file
22
packages/api/src/channels/config.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import { channelStorageConfigSchema } from 'librechat-data-provider';
|
||||
import type { ChannelStorageConfig } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { GetAppConfigOptions } from '~/app/service';
|
||||
|
||||
/** Config installs its reader during composition, without a models-to-config import cycle. */
|
||||
export function createChannelStorageConfigReader(): {
|
||||
initialize: (reader: (options?: GetAppConfigOptions) => Promise<AppConfig>) => void;
|
||||
getConfig: () => Promise<ChannelStorageConfig>;
|
||||
} {
|
||||
let readConfig: ((options?: GetAppConfigOptions) => Promise<AppConfig>) | undefined;
|
||||
return {
|
||||
initialize(reader) {
|
||||
readConfig = reader;
|
||||
},
|
||||
async getConfig() {
|
||||
if (!readConfig) throw new Error('Channel storage config reader has not been initialized');
|
||||
const config = await readConfig({ baseOnly: true });
|
||||
return channelStorageConfigSchema.parse(config.endpoints?.agents?.channels ?? {});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -155,3 +155,5 @@ export type * from './types';
|
|||
export * from './tools/approval';
|
||||
|
||||
export * from './mcp/approval';
|
||||
|
||||
export { createChannelStorageConfigReader } from './channels/config';
|
||||
|
|
|
|||
|
|
@ -574,3 +574,22 @@ describe('requirePreAuthTenantContext', () => {
|
|||
expect(next).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('reserved tenant partition', () => {
|
||||
it('rejects a reserved tenant restored from the authenticated user', () => {
|
||||
const req = mockReq({ tenantId: '_', role: 'user' });
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
tenantContextMiddleware(req, res, next);
|
||||
expect(res.status).toHaveBeenCalledWith(403);
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
});
|
||||
it('rejects the reserved tenant when a request resumes after an async boundary', async () => {
|
||||
const req = mockReq({ tenantId: '_', role: 'user' });
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
await restoreTenantContextFromReq(req, res, next);
|
||||
expect(res.status).toHaveBeenCalledWith(403);
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -184,3 +184,25 @@ describe('preAuthTenantMiddleware', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reserved tenant header boundary', () => {
|
||||
it('rejects the tenantless persistence sentinel before invoking unauthenticated handlers', () => {
|
||||
const previous = process.env.TRUST_TENANT_HEADER;
|
||||
process.env.TRUST_TENANT_HEADER = 'true';
|
||||
const response = { status: jest.fn().mockReturnThis(), json: jest.fn() };
|
||||
const next = jest.fn();
|
||||
try {
|
||||
preAuthTenantMiddleware(
|
||||
{ headers: { 'x-tenant-id': '_' } } as unknown as Request,
|
||||
response as unknown as Response,
|
||||
next,
|
||||
);
|
||||
expect(response.status).toHaveBeenCalledWith(400);
|
||||
expect(response.json).toHaveBeenCalledWith({ error: 'Reserved tenant identifier' });
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.TRUST_TENANT_HEADER;
|
||||
else process.env.TRUST_TENANT_HEADER = previous;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -50,6 +50,12 @@ export function preAuthTenantMiddleware(req: Request, res: Response, next: NextF
|
|||
}
|
||||
|
||||
const rejection = rejectTenantHeader(tenantId);
|
||||
if (rejection === 'reserved') {
|
||||
runWithTenantContext(requestContext, () =>
|
||||
res.status(400).json({ error: 'Reserved tenant identifier' }),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (rejection === 'system') {
|
||||
runWithTenantContext(requestContext, () => {
|
||||
logger.warn('[preAuthTenant] Rejected __SYSTEM__ sentinel in X-Tenant-Id header', {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
import { randomUUID } from 'crypto';
|
||||
import { unlink } from 'fs/promises';
|
||||
import { isMainThread } from 'worker_threads';
|
||||
import { tenantStorage, getTenantId, logger, SYSTEM_TENANT_ID } from '@librechat/data-schemas';
|
||||
import {
|
||||
tenantStorage,
|
||||
getTenantId,
|
||||
logger,
|
||||
SYSTEM_TENANT_ID,
|
||||
isReservedTenantId,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { Response, NextFunction, RequestHandler } from 'express';
|
||||
import type { TenantContext } from '@librechat/data-schemas';
|
||||
import type { ServerRequest } from '~/types/http';
|
||||
|
|
@ -171,11 +177,15 @@ export function tenantContextMiddleware(
|
|||
const context = buildTenantContext(req);
|
||||
const { tenantId } = context;
|
||||
|
||||
if (tenantId === SYSTEM_TENANT_ID) {
|
||||
if (tenantId === SYSTEM_TENANT_ID || isReservedTenantId(tenantId)) {
|
||||
logger.warn('[tenantContextMiddleware] Rejected system tenant for request route', {
|
||||
path: req.path,
|
||||
});
|
||||
res.status(403).json({ error: SYSTEM_TENANT_REJECTION_MESSAGE });
|
||||
res.status(403).json({
|
||||
error: isReservedTenantId(tenantId)
|
||||
? 'Reserved tenant identifier'
|
||||
: SYSTEM_TENANT_REJECTION_MESSAGE,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -305,11 +315,17 @@ export function restoreTenantContextFromReq(
|
|||
return;
|
||||
}
|
||||
|
||||
if (resolvedTenantId === SYSTEM_TENANT_ID) {
|
||||
if (resolvedTenantId === SYSTEM_TENANT_ID || isReservedTenantId(resolvedTenantId)) {
|
||||
logger.warn('[restoreTenantContextFromReq] Rejected system tenant for request route', {
|
||||
path: req.path,
|
||||
});
|
||||
return rejectRequestWithUploadCleanup(req, res, SYSTEM_TENANT_REJECTION_MESSAGE);
|
||||
return rejectRequestWithUploadCleanup(
|
||||
req,
|
||||
res,
|
||||
isReservedTenantId(resolvedTenantId)
|
||||
? 'Reserved tenant identifier'
|
||||
: SYSTEM_TENANT_REJECTION_MESSAGE,
|
||||
);
|
||||
}
|
||||
|
||||
const currentContext = tenantStorage.getStore();
|
||||
|
|
|
|||
7
packages/api/src/middleware/tenantHeader.spec.ts
Normal file
7
packages/api/src/middleware/tenantHeader.spec.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import { rejectTenantHeader } from './tenantHeader';
|
||||
|
||||
it('reserves the channel tenantless identifier without banning underscores inside real tenant IDs', () => {
|
||||
expect(rejectTenantHeader('_')).toBe('reserved');
|
||||
expect(rejectTenantHeader('org_1')).toBeUndefined();
|
||||
expect(rejectTenantHeader('__SYSTEM__')).toBe('system');
|
||||
});
|
||||
|
|
@ -1,16 +1,19 @@
|
|||
import { SYSTEM_TENANT_ID } from '@librechat/data-schemas';
|
||||
import { SYSTEM_TENANT_ID, isReservedTenantId } from '@librechat/data-schemas';
|
||||
|
||||
const MAX_TENANT_ID_LENGTH = 128;
|
||||
const VALID_TENANT_ID = /^[-a-zA-Z0-9_.]+$/;
|
||||
|
||||
/**
|
||||
* Why a trusted `X-Tenant-Id` value names no tenant: the system sentinel, or a value that is
|
||||
* Why a trusted `X-Tenant-Id` value names no tenant: a reserved persistence/system sentinel, or a value that is
|
||||
* too long or carries characters outside the tenant id alphabet. `undefined` accepts it.
|
||||
*/
|
||||
export function rejectTenantHeader(tenantId: string): 'system' | 'malformed' | undefined {
|
||||
export function rejectTenantHeader(
|
||||
tenantId: string,
|
||||
): 'system' | 'reserved' | 'malformed' | undefined {
|
||||
if (tenantId === SYSTEM_TENANT_ID) {
|
||||
return 'system';
|
||||
}
|
||||
if (isReservedTenantId(tenantId)) return 'reserved';
|
||||
if (tenantId.length > MAX_TENANT_ID_LENGTH || !VALID_TENANT_ID.test(tenantId)) {
|
||||
return 'malformed';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3371,3 +3371,33 @@ describe('directory verification deadline configuration', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('channel storage work limit', () => {
|
||||
it('preserves the claim default and exposes an operator override through YAML config', () => {
|
||||
expect(agentsEndpointSchema.parse({ channels: {} }).channels?.claimBatchSize).toBe(25);
|
||||
expect(
|
||||
configSchema.parse({
|
||||
version: '1.3.17',
|
||||
endpoints: { agents: { channels: { claimBatchSize: 60 } } },
|
||||
}).endpoints?.agents?.channels?.claimBatchSize,
|
||||
).toBe(60);
|
||||
});
|
||||
it.each([0, 101, 1.5, '25'])('rejects invalid claim batch %s', (claimBatchSize) => {
|
||||
expect(agentsEndpointSchema.safeParse({ channels: { claimBatchSize } }).success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('channel erasure configuration', () => {
|
||||
it('defaults cleanup batches and exposes a deployment override', () => {
|
||||
expect(agentsEndpointSchema.parse({ channels: {} }).channels?.erasureBatchSize).toBe(250);
|
||||
expect(
|
||||
configSchema.parse({
|
||||
version: '1.3.17',
|
||||
endpoints: { agents: { channels: { erasureBatchSize: 12 } } },
|
||||
}).endpoints?.agents?.channels?.erasureBatchSize,
|
||||
).toBe(12);
|
||||
});
|
||||
it.each([0, 1001, 1.5, '250'])('rejects invalid erasure batch %s', (erasureBatchSize) => {
|
||||
expect(agentsEndpointSchema.safeParse({ channels: { erasureBatchSize } }).success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1229,6 +1229,25 @@ const managementApiAuthSchema = z
|
|||
}
|
||||
});
|
||||
|
||||
export const CHANNEL_DELIVERY_PARTS_HARD_MAX = 100;
|
||||
|
||||
/** Storage work per claim transaction; shared by configuration and injected storage defaults. */
|
||||
export const channelStorageConfigSchema = z.object({
|
||||
erasureBatchSize: z.number().int().min(1).max(1000).default(250),
|
||||
claimBatchSize: z.number().int().min(1).max(100).default(25),
|
||||
maxDeliveryParts: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(CHANNEL_DELIVERY_PARTS_HARD_MAX)
|
||||
.default(CHANNEL_DELIVERY_PARTS_HARD_MAX),
|
||||
maxMetadataPageSize: z.number().int().min(1).max(100).default(100),
|
||||
projectionYieldMs: z.number().int().min(100).max(60_000).default(1000),
|
||||
maxPriorSummaryRows: z.number().int().min(1).max(10_000).default(1000),
|
||||
});
|
||||
|
||||
export type ChannelStorageConfig = z.infer<typeof channelStorageConfigSchema>;
|
||||
|
||||
const managementApiSchema = z.object({ auth: managementApiAuthSchema.optional() }).strict();
|
||||
|
||||
/**
|
||||
|
|
@ -2260,6 +2279,7 @@ export const agentsEndpointSchema = baseEndpointSchema
|
|||
.optional(),
|
||||
managementApi: managementApiSchema.optional(),
|
||||
remoteApi: remoteApiSchema.optional(),
|
||||
channels: channelStorageConfigSchema.optional(),
|
||||
/** Human-in-the-loop tool approval policy. Off by default. */
|
||||
toolApproval: toolApprovalPolicySchema,
|
||||
/** Ask User question behavior; see {@link askUserQuestionConfigSchema}. */
|
||||
|
|
|
|||
|
|
@ -115,4 +115,6 @@ export * from './approval';
|
|||
|
||||
export * from './types/responseReview';
|
||||
|
||||
export * from './types/channelRuns';
|
||||
|
||||
export * from './codeSetup';
|
||||
|
|
|
|||
162
packages/data-provider/src/types/channelRuns.ts
Normal file
162
packages/data-provider/src/types/channelRuns.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import type { AccountingBasis, ResponseAccounting } from './responseAccounting';
|
||||
|
||||
/** Verified transport identities are supplied by the gateway, never by Agent output. */
|
||||
export interface ChannelRunActor {
|
||||
teamId: string;
|
||||
appId: string;
|
||||
externalUserId: string;
|
||||
}
|
||||
|
||||
export interface ChannelRunPrincipal {
|
||||
userId: string;
|
||||
channelRevision: string;
|
||||
linkId?: string;
|
||||
linkRevision?: string;
|
||||
}
|
||||
|
||||
export interface ChannelRunAdmission {
|
||||
eventId: string;
|
||||
externalChannelId: string;
|
||||
externalThreadKey: string;
|
||||
externalMessageKey: string;
|
||||
actor: ChannelRunActor;
|
||||
input: string;
|
||||
}
|
||||
|
||||
export type ChannelRunState =
|
||||
| 'queued'
|
||||
| 'executing'
|
||||
| 'requires_action'
|
||||
| 'delivering'
|
||||
| 'delivered'
|
||||
| 'failed'
|
||||
| 'cancelled'
|
||||
| 'blocked';
|
||||
|
||||
export interface ChannelRunFence {
|
||||
runId: string;
|
||||
owner: string;
|
||||
epoch: number;
|
||||
}
|
||||
|
||||
export interface ChannelRunUsage {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
cachedTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
}
|
||||
|
||||
export interface ChannelSummaryMetric<T> {
|
||||
coverage: 'complete' | 'partial' | 'unavailable';
|
||||
coveredRuns: number;
|
||||
/** Present only when every expected run is covered. */
|
||||
value?: T;
|
||||
/** A known subtotal, never presented as the full thread total. */
|
||||
knownValue?: T;
|
||||
}
|
||||
|
||||
/** Current run plus prior attempts in this actor/link revision's lane, not other thread actors. */
|
||||
export interface ChannelRunSummary {
|
||||
version: 1;
|
||||
scope: 'actor_lane';
|
||||
throughSequence: number;
|
||||
includesCurrent: true;
|
||||
priorRuns: number;
|
||||
expectedRuns: number;
|
||||
observedRuns: number;
|
||||
costUSD: ChannelSummaryMetric<string> & { basis?: AccountingBasis };
|
||||
/** Sum of per-run admission-to-final durations, not the thread's wall-clock span. */
|
||||
elapsedMs: ChannelSummaryMetric<number>;
|
||||
tokens: ChannelSummaryMetric<
|
||||
Pick<ChannelRunUsage, 'inputTokens' | 'outputTokens' | 'totalTokens'>
|
||||
>;
|
||||
/** Existing authorized trace UI session; never a collector URL or a raw OAuth URL. */
|
||||
session?: { url: string };
|
||||
}
|
||||
|
||||
/** This is a safe projection, not the Agent transcript or private reasoning. */
|
||||
export interface ChannelRunSnapshot {
|
||||
recovery?: 'execution_state_unavailable';
|
||||
responseId?: string;
|
||||
text: string;
|
||||
status: 'running' | 'requires_action' | 'completed' | 'failed' | 'cancelled';
|
||||
elapsedMs: number;
|
||||
usage?: ChannelRunUsage;
|
||||
accounting?: ResponseAccounting;
|
||||
summary?: ChannelRunSummary;
|
||||
artifacts?: ChannelArtifact[];
|
||||
artifactsUnavailable?: boolean;
|
||||
activity?: 'working' | 'tool_requested' | 'tool_finished';
|
||||
pendingAction?: {
|
||||
actionId: string;
|
||||
kind: 'tool_approval' | 'ask_user_question' | 'oauth';
|
||||
expiresAt: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ChannelArtifact {
|
||||
artifactId: string;
|
||||
filename: string;
|
||||
mediaType: string;
|
||||
bytes: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
/** A deterministic render plan is immutable once its first transport operation starts. */
|
||||
export interface ChannelDeliveryPart {
|
||||
operationId: string;
|
||||
kind: 'message' | 'artifact';
|
||||
required: boolean;
|
||||
text: string;
|
||||
/** Provider-defined, versioned JSON presentation; immutable alongside the text fallback. */
|
||||
presentation?: { format: string; payload: string };
|
||||
/** Opaque server-authorized artifact identity; never an arbitrary URL or disk path. */
|
||||
artifactId?: string;
|
||||
}
|
||||
|
||||
export interface ChannelDeliveryPlan {
|
||||
revision: string;
|
||||
rendererVersion: string;
|
||||
parts: ChannelDeliveryPart[];
|
||||
}
|
||||
|
||||
export type ChannelDeliveryOutcome =
|
||||
| 'planned'
|
||||
| 'inflight'
|
||||
| 'confirmed'
|
||||
| 'retryable'
|
||||
| 'unknown'
|
||||
| 'permanent_failure';
|
||||
|
||||
export interface ChannelDeliveryReceipt {
|
||||
operationId: string;
|
||||
outcome: ChannelDeliveryOutcome;
|
||||
attempts: number;
|
||||
externalMessageKey?: string;
|
||||
externalFileId?: string;
|
||||
retryAt?: number;
|
||||
errorCode?: string;
|
||||
upload?: { fileId: string; stage: 'allocated' | 'uploaded' };
|
||||
}
|
||||
|
||||
export interface ChannelRunProgress
|
||||
extends Omit<ChannelDeliveryReceipt, 'operationId' | 'externalFileId'> {
|
||||
revision: number;
|
||||
rendererVersion: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ChannelRunView {
|
||||
runId: string;
|
||||
channelId: string;
|
||||
revision: string;
|
||||
state: ChannelRunState;
|
||||
admission: ChannelRunAdmission;
|
||||
sequence: number;
|
||||
snapshot?: ChannelRunSnapshot;
|
||||
progress?: ChannelRunProgress;
|
||||
plan?: ChannelDeliveryPlan;
|
||||
receipts: ChannelDeliveryReceipt[];
|
||||
cancellationRequested: boolean;
|
||||
}
|
||||
|
|
@ -50,3 +50,14 @@ describe('scopedCacheKey', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reserved channel partition', () => {
|
||||
it('refuses to create a named-tenant cache namespace for the tenantless sentinel', () => {
|
||||
expect(() => tenantStorage.run({ tenantId: '_' }, () => scopedCacheKey('private'))).toThrow(
|
||||
'Reserved tenant identifier',
|
||||
);
|
||||
expect(tenantStorage.run({ tenantId: 'org_1' }, () => scopedCacheKey('private'))).toBe(
|
||||
'private:org_1',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,6 +11,11 @@ export interface TenantContext {
|
|||
/** Sentinel value for deliberate cross-tenant system operations */
|
||||
export const SYSTEM_TENANT_ID = '__SYSTEM__';
|
||||
|
||||
/** Reserved for persisted tenantless channel partitions; never an authenticated tenant. */
|
||||
export function isReservedTenantId(tenantId: string | undefined): boolean {
|
||||
return tenantId === '_';
|
||||
}
|
||||
|
||||
/**
|
||||
* AsyncLocalStorage instance for propagating tenant context.
|
||||
* Callbacks passed to `tenantStorage.run()` must be `async` for the context to propagate
|
||||
|
|
@ -21,7 +26,9 @@ export const tenantStorage: AsyncLocalStorage<TenantContext> =
|
|||
|
||||
/** Returns the current tenant ID from async context, or undefined if none is set */
|
||||
export function getTenantId(): string | undefined {
|
||||
return tenantStorage.getStore()?.tenantId;
|
||||
const tenantId = tenantStorage.getStore()?.tenantId;
|
||||
if (isReservedTenantId(tenantId)) throw new TypeError('Reserved tenant identifier');
|
||||
return tenantId;
|
||||
}
|
||||
|
||||
/** Returns the current user ID from async context, or undefined if none is set */
|
||||
|
|
|
|||
|
|
@ -29,6 +29,10 @@ export type {
|
|||
} from './methods/response';
|
||||
export {
|
||||
createMethods,
|
||||
createChannelRunStorage,
|
||||
createChannelSummaryStorage,
|
||||
CHANNEL_SUMMARY_MAX_PRIOR_ROWS,
|
||||
ChannelRunConflict,
|
||||
createSkillDeletionPersistenceAdapter,
|
||||
AGENT_OWNER_CONTACT_RESOLVED_FIELD,
|
||||
AgentSortCursorError,
|
||||
|
|
@ -133,6 +137,7 @@ export {
|
|||
runAsSystem,
|
||||
scopedCacheKey,
|
||||
SYSTEM_TENANT_ID,
|
||||
isReservedTenantId,
|
||||
} from './config/tenantContext';
|
||||
export type { TenantContext } from './config/tenantContext';
|
||||
export {
|
||||
|
|
|
|||
41
packages/data-schemas/src/methods/channelErasure.ts
Normal file
41
packages/data-schemas/src/methods/channelErasure.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { channelStorageConfigSchema } from 'librechat-data-provider';
|
||||
import type { ChannelStorageConfig } from 'librechat-data-provider';
|
||||
import type { ResponseOwner } from '~/models/response';
|
||||
import { createChannelRunModel, createChannelLaneModel } from '~/models/channelRun';
|
||||
|
||||
/** Validate operation configuration before the caller closes irreversible admission fences. */
|
||||
export function resolveChannelErasureConfig(
|
||||
options: Partial<ChannelStorageConfig> = {},
|
||||
): ChannelStorageConfig {
|
||||
return channelStorageConfigSchema.parse(options);
|
||||
}
|
||||
|
||||
/** The caller closes the response owner admission anchor before entering this cascade. */
|
||||
export async function eraseChannelPersistenceForUser(
|
||||
mongoose: typeof import('mongoose'),
|
||||
owner: ResponseOwner,
|
||||
options: Pick<Partial<ChannelStorageConfig>, 'erasureBatchSize'> = {},
|
||||
): Promise<void> {
|
||||
const { erasureBatchSize } = resolveChannelErasureConfig(options);
|
||||
const tenantId = owner.tenantId || '_';
|
||||
const runs = createChannelRunModel(mongoose);
|
||||
const lanes = createChannelLaneModel(mongoose);
|
||||
const filter = { tenantId, 'principal.userId': owner.userId };
|
||||
// New lanes have an indexed owner. Older rows are recovered from the still-retained runs.
|
||||
await lanes.deleteMany({ tenantId, userId: owner.userId });
|
||||
const cursor = runs.find(filter).select('laneId').lean().cursor();
|
||||
const batch = new Set<string>();
|
||||
try {
|
||||
for await (const run of cursor) {
|
||||
batch.add(run.laneId);
|
||||
if (batch.size === erasureBatchSize) {
|
||||
await lanes.deleteMany({ tenantId, laneId: { $in: [...batch] } });
|
||||
batch.clear();
|
||||
}
|
||||
}
|
||||
if (batch.size) await lanes.deleteMany({ tenantId, laneId: { $in: [...batch] } });
|
||||
} finally {
|
||||
await cursor.close();
|
||||
}
|
||||
await runs.deleteMany(filter);
|
||||
}
|
||||
37
packages/data-schemas/src/methods/channelIndexes.spec.ts
Normal file
37
packages/data-schemas/src/methods/channelIndexes.spec.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { createChannelIndexGate } from './channelIndexes';
|
||||
|
||||
it('serializes required index builds and shares the readiness promise across concurrent admissions', async () => {
|
||||
const order: string[] = [];
|
||||
const first = {
|
||||
modelName: 'First',
|
||||
createIndexes: jest.fn(async () => {
|
||||
order.push('first');
|
||||
}),
|
||||
};
|
||||
const second = {
|
||||
modelName: 'Second',
|
||||
createIndexes: jest.fn(async () => {
|
||||
order.push('second');
|
||||
}),
|
||||
};
|
||||
const ready = createChannelIndexGate(() => [first, second]);
|
||||
await Promise.all([ready(), ready(), ready()]);
|
||||
await ready();
|
||||
expect(order).toEqual(['first', 'second']);
|
||||
expect(first.createIndexes).toHaveBeenCalledTimes(1);
|
||||
expect(second.createIndexes).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('fails closed on index failure and permits a later readiness retry', async () => {
|
||||
const model = {
|
||||
modelName: 'ChannelRun',
|
||||
createIndexes: jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('index rejected'))
|
||||
.mockResolvedValue(undefined),
|
||||
};
|
||||
const ready = createChannelIndexGate(() => [model]);
|
||||
await expect(ready()).rejects.toThrow('index rejected');
|
||||
await expect(ready()).resolves.toBeUndefined();
|
||||
expect(model.createIndexes).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
19
packages/data-schemas/src/methods/channelIndexes.ts
Normal file
19
packages/data-schemas/src/methods/channelIndexes.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { createIndexesWithRetry } from '~/utils/retry';
|
||||
|
||||
/** Unique indexes are correctness fences, even when automatic indexing is disabled. */
|
||||
export function createChannelIndexGate(
|
||||
models: () => Array<Parameters<typeof createIndexesWithRetry>[0]>,
|
||||
afterReady?: () => Promise<void>,
|
||||
): () => Promise<void> {
|
||||
let pending: Promise<void> | undefined;
|
||||
return () => {
|
||||
pending ??= (async () => {
|
||||
for (const model of models()) await createIndexesWithRetry(model);
|
||||
await afterReady?.();
|
||||
})().catch((error: unknown) => {
|
||||
pending = undefined;
|
||||
throw error;
|
||||
});
|
||||
return pending;
|
||||
};
|
||||
}
|
||||
29
packages/data-schemas/src/methods/channelOwner.ts
Normal file
29
packages/data-schemas/src/methods/channelOwner.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import type { ClientSession } from 'mongoose';
|
||||
import type { ResponseOwner } from '~/models/response';
|
||||
import { responseScopeKeys, initializeResponseScopes } from './responseScope';
|
||||
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
|
||||
import { createResponseScopeModel } from '~/models/responseScope';
|
||||
|
||||
/** All private channel persistence joins the same account-erasure admission fence. */
|
||||
export async function prepareChannelOwnerAdmission(
|
||||
mongoose: typeof import('mongoose'),
|
||||
owner: ResponseOwner,
|
||||
): Promise<(session: ClientSession) => Promise<boolean>> {
|
||||
const current = getTenantId() || undefined;
|
||||
if (
|
||||
!owner.userId ||
|
||||
owner.tenantId === SYSTEM_TENANT_ID ||
|
||||
(current && current !== SYSTEM_TENANT_ID && current !== (owner.tenantId || undefined))
|
||||
)
|
||||
throw new TypeError('Channel owner does not match authenticated scope');
|
||||
const [key] = responseScopeKeys(owner, '');
|
||||
await initializeResponseScopes(mongoose, [key]);
|
||||
return async (session) => {
|
||||
const admitted = await createResponseScopeModel(mongoose).updateOne(
|
||||
{ _id: key, closed: false },
|
||||
{ $inc: { revision: 1 } },
|
||||
{ session },
|
||||
);
|
||||
return admitted.matchedCount === 1;
|
||||
};
|
||||
}
|
||||
133
packages/data-schemas/src/methods/channelProtocol.spec.ts
Normal file
133
packages/data-schemas/src/methods/channelProtocol.spec.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import type { ChannelDeliveryOutcome, ChannelRunSnapshot } from 'librechat-data-provider';
|
||||
import type { ChannelRunRecord } from '~/models/channelRun';
|
||||
import { deliveryDisposition, snapshotPatch } from './channelProtocol';
|
||||
|
||||
function run(): ChannelRunRecord {
|
||||
return {
|
||||
tenantId: 'tenant',
|
||||
channelId: 'channel',
|
||||
agentId: 'agent',
|
||||
laneId: 'lane',
|
||||
runId: 'run',
|
||||
principal: { userId: 'user', channelRevision: 'revision' },
|
||||
payloadHash: 'hash',
|
||||
sequence: 1,
|
||||
admission: {
|
||||
eventId: 'event',
|
||||
externalChannelId: 'C',
|
||||
externalThreadKey: 'C:thread',
|
||||
externalMessageKey: 'C:event',
|
||||
actor: { teamId: 'team', appId: 'app', externalUserId: 'actor' },
|
||||
input: 'question',
|
||||
},
|
||||
receivedAt: 1,
|
||||
state: 'delivering',
|
||||
leaseEpoch: 1,
|
||||
leaseExpiresAt: 1000,
|
||||
cancellationRequested: false,
|
||||
execution: {
|
||||
requestId: 'run',
|
||||
conversationId: 'conversation',
|
||||
parentMessageId: 'parent',
|
||||
responseId: 'response',
|
||||
},
|
||||
snapshot: { status: 'completed', responseId: 'response', text: 'answer', elapsedMs: 1 },
|
||||
plan: {
|
||||
revision: 'plan',
|
||||
rendererVersion: 'test',
|
||||
parts: [
|
||||
{ operationId: 'required', kind: 'message', required: true, text: 'answer' },
|
||||
{ operationId: 'optional', kind: 'message', required: false, text: 'footer' },
|
||||
],
|
||||
},
|
||||
receipts: [
|
||||
{
|
||||
operationId: 'required',
|
||||
outcome: 'confirmed',
|
||||
attempts: 1,
|
||||
externalMessageKey: 'C:required',
|
||||
},
|
||||
{
|
||||
operationId: 'optional',
|
||||
outcome: 'confirmed',
|
||||
attempts: 1,
|
||||
externalMessageKey: 'C:optional',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const outcomes: ChannelDeliveryOutcome[] = [
|
||||
'planned',
|
||||
'inflight',
|
||||
'confirmed',
|
||||
'retryable',
|
||||
'unknown',
|
||||
'permanent_failure',
|
||||
];
|
||||
const deliveryCases = outcomes.flatMap((required) =>
|
||||
outcomes.map((optional) => ({ required, optional })),
|
||||
);
|
||||
it.each(deliveryCases)(
|
||||
'finalizes required=$required and optional=$optional according to delivery policy',
|
||||
({ required, optional }) => {
|
||||
const current = run();
|
||||
current.receipts[0].outcome = required;
|
||||
current.receipts[1].outcome = optional;
|
||||
const canFinish =
|
||||
![required, optional].includes('inflight') &&
|
||||
['confirmed', 'permanent_failure'].includes(required);
|
||||
if (!canFinish) expect(() => deliveryDisposition(current, 100)).toThrow('delivery_incomplete');
|
||||
else
|
||||
expect(deliveryDisposition(current, 100)).toBe(
|
||||
required === 'permanent_failure' ? 'failed' : 'delivered',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const statuses: ChannelRunSnapshot['status'][] = [
|
||||
'running',
|
||||
'requires_action',
|
||||
'completed',
|
||||
'failed',
|
||||
'cancelled',
|
||||
];
|
||||
const snapshotCases = statuses.flatMap((previous) => statuses.map((next) => ({ previous, next })));
|
||||
it.each(snapshotCases)(
|
||||
'projects $previous to $next without reopening terminal work',
|
||||
({ previous, next }) => {
|
||||
const current = run();
|
||||
delete current.plan;
|
||||
current.snapshot = { ...current.snapshot!, status: previous };
|
||||
const phases: Record<ChannelRunSnapshot['status'], ChannelRunRecord['state']> = {
|
||||
running: 'executing',
|
||||
requires_action: 'requires_action',
|
||||
completed: 'delivering',
|
||||
failed: 'delivering',
|
||||
cancelled: 'delivering',
|
||||
};
|
||||
current.state = phases[previous];
|
||||
const incoming = { ...current.snapshot, status: next };
|
||||
if (['completed', 'failed', 'cancelled'].includes(previous) && previous !== next)
|
||||
expect(() => snapshotPatch(current, incoming)).toThrow('invalid_transition');
|
||||
else expect(snapshotPatch(current, incoming).state).toBe(phases[next]);
|
||||
},
|
||||
);
|
||||
|
||||
it('does not finalize while a progress operation is still active', () => {
|
||||
const current = run();
|
||||
current.progress = {
|
||||
revision: 1,
|
||||
rendererVersion: 'test',
|
||||
text: 'progress',
|
||||
outcome: 'inflight',
|
||||
attempts: 1,
|
||||
};
|
||||
expect(() => deliveryDisposition(current, 100)).toThrow('delivery_incomplete');
|
||||
});
|
||||
|
||||
it('rejects duplicate receipt identities instead of treating an incomplete plan as confirmed', () => {
|
||||
const current = run();
|
||||
current.receipts[1] = current.receipts[0];
|
||||
expect(() => deliveryDisposition(current, 100)).toThrow('invalid_transition');
|
||||
});
|
||||
240
packages/data-schemas/src/methods/channelProtocol.ts
Normal file
240
packages/data-schemas/src/methods/channelProtocol.ts
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type {
|
||||
ChannelRunAdmission,
|
||||
ChannelRunPrincipal,
|
||||
ChannelRunSnapshot,
|
||||
ChannelDeliveryPlan,
|
||||
ChannelDeliveryReceipt,
|
||||
} from 'librechat-data-provider';
|
||||
import type { ChannelRunRecord } from '~/models/channelRun';
|
||||
|
||||
export interface ChannelLaneScope {
|
||||
tenantId: string;
|
||||
channelId: string;
|
||||
agentId: string;
|
||||
principal: ChannelRunPrincipal;
|
||||
admission: Pick<ChannelRunAdmission, 'actor' | 'externalChannelId' | 'externalThreadKey'>;
|
||||
}
|
||||
|
||||
export interface ChannelRunEnrollment extends ChannelLaneScope {
|
||||
admission: ChannelRunAdmission;
|
||||
}
|
||||
|
||||
export class ChannelRunConflict extends Error {
|
||||
constructor(
|
||||
readonly code: 'event_conflict' | 'stale_fence' | 'invalid_transition' | 'delivery_incomplete',
|
||||
) {
|
||||
super(code);
|
||||
this.name = 'ChannelRunConflict';
|
||||
}
|
||||
}
|
||||
|
||||
export const digest = (value: object): string =>
|
||||
createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify(value, (_key, item: unknown) =>
|
||||
item != null && typeof item === 'object' && !Array.isArray(item)
|
||||
? Object.fromEntries(
|
||||
Object.entries(item).sort(([a], [b]) => {
|
||||
if (a === b) return 0;
|
||||
return a < b ? -1 : 1;
|
||||
}),
|
||||
)
|
||||
: item,
|
||||
),
|
||||
)
|
||||
.digest('hex');
|
||||
export function laneKey(scope: ChannelLaneScope): string {
|
||||
return digest({
|
||||
tenantId: scope.tenantId,
|
||||
channelId: scope.channelId,
|
||||
agentId: scope.agentId,
|
||||
thread: scope.admission.externalThreadKey,
|
||||
externalChannelId: scope.admission.externalChannelId,
|
||||
actor: scope.admission.actor,
|
||||
userId: scope.principal.userId,
|
||||
revision: scope.principal.channelRevision,
|
||||
linkRevision: scope.principal.linkRevision ?? null,
|
||||
linkId: scope.principal.linkId ?? null,
|
||||
});
|
||||
}
|
||||
export const activeStates: string[] = [
|
||||
'queued',
|
||||
'executing',
|
||||
'requires_action',
|
||||
'delivering',
|
||||
'blocked',
|
||||
];
|
||||
|
||||
/** A projection cannot grant a different response authority to the next actor-lane turn. */
|
||||
export function assertSnapshotBinding(run: ChannelRunRecord, snapshot: ChannelRunSnapshot): void {
|
||||
if (
|
||||
(snapshot.status === 'completed' && !snapshot.responseId) ||
|
||||
(snapshot.responseId != null && snapshot.responseId !== run.execution?.responseId)
|
||||
)
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
}
|
||||
|
||||
/** Confirm the provider identity appropriate to the immutable plan, on every confirmation path. */
|
||||
export function assertDeliveryEvidence(
|
||||
run: ChannelRunRecord,
|
||||
receipt: ChannelDeliveryReceipt,
|
||||
previous: ChannelDeliveryReceipt | undefined,
|
||||
): void {
|
||||
const part = run.plan?.parts.find((item) => item.operationId === receipt.operationId);
|
||||
if (
|
||||
!part ||
|
||||
(part.kind === 'artifact' &&
|
||||
(!part.artifactId ||
|
||||
!run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))) ||
|
||||
!previous ||
|
||||
(previous.externalMessageKey != null &&
|
||||
receipt.externalMessageKey != null &&
|
||||
previous.externalMessageKey !== receipt.externalMessageKey) ||
|
||||
(previous.externalFileId != null &&
|
||||
receipt.externalFileId != null &&
|
||||
previous.externalFileId !== receipt.externalFileId) ||
|
||||
(receipt.upload != null && digest(receipt.upload) !== digest(previous.upload ?? {})) ||
|
||||
(receipt.externalFileId != null &&
|
||||
previous.upload != null &&
|
||||
previous.upload.fileId !== receipt.externalFileId) ||
|
||||
(receipt.outcome === 'confirmed' &&
|
||||
((part.kind === 'message' && !receipt.externalMessageKey) ||
|
||||
(part.kind === 'artifact' &&
|
||||
(!receipt.externalFileId ||
|
||||
(previous.upload != null && previous.upload.fileId !== receipt.externalFileId)))))
|
||||
)
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
}
|
||||
|
||||
export function leaseDeadline(now: number, leaseMs: number): number {
|
||||
if (
|
||||
!Number.isSafeInteger(now) ||
|
||||
!Number.isSafeInteger(leaseMs) ||
|
||||
leaseMs <= 0 ||
|
||||
!Number.isSafeInteger(now + leaseMs)
|
||||
)
|
||||
throw new Error('Invalid channel lease');
|
||||
return now + leaseMs;
|
||||
}
|
||||
|
||||
/** Evidence is verified by the transport; storage binds it to the exact uncertain attempt. */
|
||||
export function assertReconciliation(
|
||||
previous: Omit<ChannelDeliveryReceipt, 'operationId'> | undefined,
|
||||
receipt: Pick<ChannelDeliveryReceipt, 'attempts' | 'externalMessageKey' | 'externalFileId'>,
|
||||
): void {
|
||||
if (
|
||||
!previous ||
|
||||
!['unknown', 'confirmed'].includes(previous.outcome) ||
|
||||
previous.attempts !== receipt.attempts ||
|
||||
(!receipt.externalMessageKey && !receipt.externalFileId) ||
|
||||
(previous.externalMessageKey && previous.externalMessageKey !== receipt.externalMessageKey) ||
|
||||
(previous.externalFileId && previous.externalFileId !== receipt.externalFileId) ||
|
||||
(previous.upload && previous.upload.fileId !== receipt.externalFileId) ||
|
||||
(previous.outcome === 'confirmed' &&
|
||||
(previous.externalMessageKey !== receipt.externalMessageKey ||
|
||||
previous.externalFileId !== receipt.externalFileId))
|
||||
)
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
}
|
||||
|
||||
const terminalSnapshots = ['completed', 'failed', 'cancelled'];
|
||||
|
||||
export function snapshotPatch(
|
||||
run: ChannelRunRecord,
|
||||
snapshot: ChannelRunSnapshot,
|
||||
): Pick<ChannelRunRecord, 'state' | 'snapshot'> {
|
||||
assertSnapshotBinding(run, snapshot);
|
||||
if (
|
||||
run.snapshot &&
|
||||
terminalSnapshots.includes(run.snapshot.status) &&
|
||||
(run.snapshot.status !== snapshot.status ||
|
||||
run.snapshot.responseId !== snapshot.responseId ||
|
||||
run.snapshot.text !== snapshot.text)
|
||||
)
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
if (run.state === 'delivering' && !terminalSnapshots.includes(snapshot.status))
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
let state: ChannelRunRecord['state'] = 'delivering';
|
||||
if (snapshot.status === 'running') state = 'executing';
|
||||
else if (snapshot.status === 'requires_action') state = 'requires_action';
|
||||
const patch: Pick<ChannelRunRecord, 'state' | 'snapshot'> = {
|
||||
state,
|
||||
snapshot:
|
||||
snapshot.summary == null && run.snapshot?.summary
|
||||
? { ...snapshot, summary: run.snapshot.summary }
|
||||
: snapshot,
|
||||
};
|
||||
if (run.plan && digest(patch.snapshot ?? {}) !== digest(run.snapshot ?? {}))
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
return patch;
|
||||
}
|
||||
|
||||
export function assertPlan(
|
||||
run: ChannelRunRecord,
|
||||
plan: ChannelDeliveryPlan,
|
||||
maxParts: number,
|
||||
): void {
|
||||
if (
|
||||
!run.snapshot ||
|
||||
!terminalSnapshots.includes(run.snapshot.status) ||
|
||||
run.state !== 'delivering' ||
|
||||
!plan.parts.length ||
|
||||
plan.parts.length > maxParts ||
|
||||
new Set(plan.parts.map((part) => part.operationId)).size !== plan.parts.length ||
|
||||
!plan.parts.some((part) => part.required) ||
|
||||
plan.parts.some(
|
||||
(part) =>
|
||||
!part.operationId ||
|
||||
!['message', 'artifact'].includes(part.kind) ||
|
||||
(part.kind === 'artifact' &&
|
||||
(!part.artifactId ||
|
||||
!run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))),
|
||||
)
|
||||
)
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
assertSnapshotBinding(run, run.snapshot);
|
||||
}
|
||||
|
||||
export function hasPermanentDeliveryFailure(run: ChannelRunRecord): boolean {
|
||||
return (
|
||||
run.plan?.parts.some(
|
||||
(part) =>
|
||||
part.required &&
|
||||
run.receipts.some(
|
||||
(receipt) =>
|
||||
receipt.operationId === part.operationId && receipt.outcome === 'permanent_failure',
|
||||
),
|
||||
) ?? false
|
||||
);
|
||||
}
|
||||
|
||||
/** A permanent required failure is terminal even though successful delivery is impossible.
|
||||
* Unknown receipts never become eligible for resend. Optional unknown effects remain best effort. */
|
||||
export function deliveryDisposition(
|
||||
run: ChannelRunRecord,
|
||||
maxParts: number,
|
||||
): 'delivered' | 'failed' | 'cancelled' {
|
||||
if (!run.plan || !run.snapshot) throw new ChannelRunConflict('delivery_incomplete');
|
||||
assertPlan(run, run.plan, maxParts);
|
||||
if (
|
||||
run.receipts.length !== run.plan.parts.length ||
|
||||
new Set(run.receipts.map((receipt) => receipt.operationId)).size !== run.receipts.length
|
||||
)
|
||||
throw new ChannelRunConflict('invalid_transition');
|
||||
for (const receipt of run.receipts) assertDeliveryEvidence(run, receipt, receipt);
|
||||
if (
|
||||
run.receipts.some((receipt) => receipt.outcome === 'inflight') ||
|
||||
run.progress?.outcome === 'inflight'
|
||||
)
|
||||
throw new ChannelRunConflict('delivery_incomplete');
|
||||
const required = run.plan.parts
|
||||
.filter((part) => part.required)
|
||||
.map((part) => run.receipts.find((receipt) => receipt.operationId === part.operationId));
|
||||
if (hasPermanentDeliveryFailure(run)) return 'failed';
|
||||
if (required.some((receipt) => receipt?.outcome !== 'confirmed'))
|
||||
throw new ChannelRunConflict('delivery_incomplete');
|
||||
if (run.snapshot.status === 'completed') return 'delivered';
|
||||
if (run.snapshot.status === 'cancelled') return 'cancelled';
|
||||
return 'failed';
|
||||
}
|
||||
1293
packages/data-schemas/src/methods/channelRun.spec.ts
Normal file
1293
packages/data-schemas/src/methods/channelRun.spec.ts
Normal file
File diff suppressed because it is too large
Load diff
1072
packages/data-schemas/src/methods/channelRun.ts
Normal file
1072
packages/data-schemas/src/methods/channelRun.ts
Normal file
File diff suppressed because it is too large
Load diff
189
packages/data-schemas/src/methods/channelSummary.spec.ts
Normal file
189
packages/data-schemas/src/methods/channelSummary.spec.ts
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import type { ChannelRunRecord } from '~/models/channelRun';
|
||||
import type { ChannelSummaryRow } from './channelSummary';
|
||||
import { createChannelSummaryStorage, CHANNEL_SUMMARY_MAX_PRIOR_ROWS } from './channelSummary';
|
||||
import { runAsSystem, tenantStorage } from '~/config/tenantContext';
|
||||
import { createChannelRunModel } from '~/models/channelRun';
|
||||
|
||||
let mongo: MongoMemoryServer;
|
||||
const storage = createChannelSummaryStorage(mongoose);
|
||||
function record(sequence = 1): ChannelRunRecord {
|
||||
return {
|
||||
tenantId: 'tenant',
|
||||
channelId: 'channel',
|
||||
agentId: 'agent',
|
||||
laneId: 'lane',
|
||||
runId: `run-${sequence}`,
|
||||
sequence,
|
||||
state: 'delivered',
|
||||
receivedAt: 1,
|
||||
payloadHash: 'hash',
|
||||
leaseEpoch: 1,
|
||||
leaseExpiresAt: 0,
|
||||
receipts: [],
|
||||
cancellationRequested: false,
|
||||
principal: {
|
||||
userId: 'same-internal-user',
|
||||
channelRevision: 'r1',
|
||||
linkId: 'link',
|
||||
linkRevision: 'l1',
|
||||
},
|
||||
admission: {
|
||||
eventId: `event-${sequence}`,
|
||||
externalChannelId: 'C1',
|
||||
externalThreadKey: 'C1:123',
|
||||
externalMessageKey: `C1:${sequence}`,
|
||||
input: 'secret transcript',
|
||||
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'ACTOR-A' },
|
||||
},
|
||||
snapshot: {
|
||||
status: 'completed',
|
||||
text: 'secret answer',
|
||||
elapsedMs: 10,
|
||||
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
|
||||
accounting: {
|
||||
status: 'available',
|
||||
currency: 'USD',
|
||||
amount: '0.1',
|
||||
basis: 'application_billing',
|
||||
coverage: 'complete',
|
||||
reason: 'priced',
|
||||
calls: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
async function read(scope: ChannelRunRecord): Promise<ChannelSummaryRow[]> {
|
||||
return tenantStorage.run({ tenantId: scope.tenantId }, async () => {
|
||||
const rows: ChannelSummaryRow[] = [];
|
||||
for await (const row of storage.priorMetrics(scope)) rows.push(row);
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
|
||||
await mongoose.connect(mongo.getUri());
|
||||
createChannelRunModel(mongoose);
|
||||
await mongoose.models.ChannelRun.init();
|
||||
}, 60000);
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongo?.stop();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await runAsSystem(async () => mongoose.models.ChannelRun.deleteMany({}));
|
||||
});
|
||||
|
||||
it('reads newest prior metrics only, using the existing lane index, without transcripts', async () => {
|
||||
await runAsSystem(async () =>
|
||||
mongoose.models.ChannelRun.insertMany([record(1), record(2), record(3)]),
|
||||
);
|
||||
const rows = await read(record(3));
|
||||
expect(rows.map((row) => row.sequence)).toEqual([2, 1]);
|
||||
expect(rows[0]).toEqual({
|
||||
sequence: 2,
|
||||
snapshot: {
|
||||
status: 'completed',
|
||||
elapsedMs: 10,
|
||||
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
|
||||
accounting: {
|
||||
status: 'available',
|
||||
currency: 'USD',
|
||||
amount: '0.1',
|
||||
basis: 'application_billing',
|
||||
coverage: 'complete',
|
||||
reason: 'priced',
|
||||
calls: [],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(rows)).not.toContain('secret');
|
||||
expect(JSON.stringify(rows)).not.toContain('same-internal-user');
|
||||
const indexes = await mongoose.models.ChannelRun.collection.indexes();
|
||||
expect(
|
||||
indexes.some(
|
||||
(index) =>
|
||||
JSON.stringify(index.key) === JSON.stringify({ tenantId: 1, laneId: 1, sequence: 1 }),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'tenant',
|
||||
'lane',
|
||||
'channel',
|
||||
'agent',
|
||||
'user',
|
||||
'channelRevision',
|
||||
'link',
|
||||
'linkRevision',
|
||||
'team',
|
||||
'app',
|
||||
'actor',
|
||||
'thread',
|
||||
'externalChannel',
|
||||
])('excludes a foreign %s even when a malformed row reuses the same lane', async (kind) => {
|
||||
const foreign = record(1);
|
||||
if (kind === 'tenant') foreign.tenantId = 'other';
|
||||
if (kind === 'lane') foreign.laneId = 'other';
|
||||
if (kind === 'channel') foreign.channelId = 'other';
|
||||
if (kind === 'agent') foreign.agentId = 'other';
|
||||
if (kind === 'user') foreign.principal.userId = 'other';
|
||||
if (kind === 'channelRevision') foreign.principal.channelRevision = 'other';
|
||||
if (kind === 'link') foreign.principal.linkId = 'other';
|
||||
if (kind === 'linkRevision') foreign.principal.linkRevision = 'other';
|
||||
if (kind === 'team') foreign.admission.actor.teamId = 'other';
|
||||
if (kind === 'app') foreign.admission.actor.appId = 'other';
|
||||
if (kind === 'actor') foreign.admission.actor.externalUserId = 'ACTOR-B';
|
||||
if (kind === 'thread') foreign.admission.externalThreadKey = 'other';
|
||||
if (kind === 'externalChannel') foreign.admission.externalChannelId = 'other';
|
||||
await runAsSystem(async () => mongoose.models.ChannelRun.create(foreign));
|
||||
expect(await read(record(2))).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps failed/cancelled/legacy attempts visible as metric coverage gaps', async () => {
|
||||
await runAsSystem(async () =>
|
||||
mongoose.models.ChannelRun.insertMany([
|
||||
{ ...record(1), state: 'failed', snapshot: undefined },
|
||||
{ ...record(2), state: 'cancelled' },
|
||||
{ ...record(3), state: 'blocked' },
|
||||
]),
|
||||
);
|
||||
expect((await read(record(4))).map((row) => row.sequence)).toEqual([3, 2, 1]);
|
||||
});
|
||||
|
||||
it('bounds long-history reads at newest 1000 rows without silently altering the sequence cutoff', async () => {
|
||||
const count = CHANNEL_SUMMARY_MAX_PRIOR_ROWS + 3;
|
||||
await runAsSystem(async () =>
|
||||
mongoose.models.ChannelRun.insertMany(Array.from({ length: count }, (_, i) => record(i + 1))),
|
||||
);
|
||||
const rows = await read(record(count));
|
||||
expect(rows).toHaveLength(CHANNEL_SUMMARY_MAX_PRIOR_ROWS);
|
||||
expect(rows[0].sequence).toBe(count - 1);
|
||||
expect(rows[rows.length - 1]?.sequence).toBe(count - CHANNEL_SUMMARY_MAX_PRIOR_ROWS);
|
||||
});
|
||||
|
||||
it('throws for invalid sequence and database failure instead of disguising outages as absence', async () => {
|
||||
await expect(read(record(0))).rejects.toThrow('Invalid channel summary sequence');
|
||||
const other = new mongoose.Mongoose();
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const row of createChannelSummaryStorage(other).priorMetrics(record(2))) void row;
|
||||
})(),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('uses the configured history work bound while retaining the newest prior rows', async () => {
|
||||
await runAsSystem(async () =>
|
||||
mongoose.models.ChannelRun.insertMany(Array.from({ length: 5 }, (_, i) => record(i + 1))),
|
||||
);
|
||||
const configured = createChannelSummaryStorage(mongoose, { maxPriorSummaryRows: 2 });
|
||||
const rows = await tenantStorage.run({ tenantId: 'tenant' }, async () => {
|
||||
const result: ChannelSummaryRow[] = [];
|
||||
for await (const row of configured.priorMetrics({ ...record(6), sequence: 6 }))
|
||||
result.push(row);
|
||||
return result;
|
||||
});
|
||||
expect(rows.map((row) => row.sequence)).toEqual([5, 4]);
|
||||
});
|
||||
86
packages/data-schemas/src/methods/channelSummary.ts
Normal file
86
packages/data-schemas/src/methods/channelSummary.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { channelStorageConfigSchema } from 'librechat-data-provider';
|
||||
import type {
|
||||
ChannelRunSnapshot,
|
||||
ResponseAccounting,
|
||||
ChannelStorageConfig,
|
||||
} from 'librechat-data-provider';
|
||||
import type { ChannelRunRecord } from '~/models/channelRun';
|
||||
|
||||
export type ChannelSummaryScope = Pick<
|
||||
ChannelRunRecord,
|
||||
'tenantId' | 'channelId' | 'agentId' | 'laneId' | 'principal' | 'sequence' | 'admission'
|
||||
>;
|
||||
|
||||
export interface ChannelSummaryRow {
|
||||
sequence: number;
|
||||
snapshot?: Pick<ChannelRunSnapshot, 'status' | 'elapsedMs' | 'usage'> & {
|
||||
accounting?: Pick<
|
||||
ResponseAccounting,
|
||||
'status' | 'currency' | 'coverage' | 'amount' | 'basis' | 'reason'
|
||||
> & { calls?: Array<Pick<ResponseAccounting['calls'][number], 'reason'>> };
|
||||
};
|
||||
}
|
||||
|
||||
export interface ChannelSummaryStorage {
|
||||
/** Newest prior attempts, with a fixed presentation safety bound; not a billing ledger. */
|
||||
priorMetrics(scope: ChannelSummaryScope): AsyncIterable<ChannelSummaryRow>;
|
||||
}
|
||||
|
||||
/** Bounds footer work independently of thread length. Older rows become explicit coverage gaps. */
|
||||
export const CHANNEL_SUMMARY_MAX_PRIOR_ROWS: number = channelStorageConfigSchema.parse(
|
||||
{},
|
||||
).maxPriorSummaryRows;
|
||||
|
||||
export function createChannelSummaryStorage(
|
||||
mongoose: typeof import('mongoose'),
|
||||
options: Pick<Partial<ChannelStorageConfig>, 'maxPriorSummaryRows'> = {},
|
||||
): ChannelSummaryStorage {
|
||||
const { maxPriorSummaryRows } = channelStorageConfigSchema.parse(options);
|
||||
return {
|
||||
async *priorMetrics(scope) {
|
||||
if (!Number.isSafeInteger(scope.sequence) || scope.sequence < 1)
|
||||
throw new Error('Invalid channel summary sequence');
|
||||
const cursor = mongoose
|
||||
.model<ChannelRunRecord>('ChannelRun')
|
||||
.find({
|
||||
tenantId: scope.tenantId,
|
||||
laneId: scope.laneId,
|
||||
sequence: { $gte: 1, $lt: scope.sequence },
|
||||
channelId: scope.channelId,
|
||||
agentId: scope.agentId,
|
||||
'principal.userId': scope.principal.userId,
|
||||
'principal.channelRevision': scope.principal.channelRevision,
|
||||
'principal.linkId': scope.principal.linkId ?? null,
|
||||
'principal.linkRevision': scope.principal.linkRevision ?? null,
|
||||
'admission.actor.teamId': scope.admission.actor.teamId,
|
||||
'admission.actor.appId': scope.admission.actor.appId,
|
||||
'admission.actor.externalUserId': scope.admission.actor.externalUserId,
|
||||
'admission.externalChannelId': scope.admission.externalChannelId,
|
||||
'admission.externalThreadKey': scope.admission.externalThreadKey,
|
||||
})
|
||||
.select({
|
||||
_id: 0,
|
||||
sequence: 1,
|
||||
'snapshot.status': 1,
|
||||
'snapshot.elapsedMs': 1,
|
||||
'snapshot.usage': 1,
|
||||
'snapshot.accounting.status': 1,
|
||||
'snapshot.accounting.currency': 1,
|
||||
'snapshot.accounting.coverage': 1,
|
||||
'snapshot.accounting.amount': 1,
|
||||
'snapshot.accounting.basis': 1,
|
||||
'snapshot.accounting.reason': 1,
|
||||
'snapshot.accounting.calls.reason': 1,
|
||||
})
|
||||
.sort({ sequence: -1 })
|
||||
.limit(maxPriorSummaryRows)
|
||||
.lean<ChannelSummaryRow[]>()
|
||||
.cursor();
|
||||
try {
|
||||
for await (const row of cursor) yield row;
|
||||
} finally {
|
||||
await cursor.close();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ export type {
|
|||
PermissionPrincipalRecord,
|
||||
PermissionPrincipalData,
|
||||
} from './provisioning';
|
||||
import type { ChannelStorageConfig } from 'librechat-data-provider';
|
||||
import { createSubagentContentMethods, type SubagentContentMethods } from './subagentContent';
|
||||
export type {
|
||||
SubagentContentScope,
|
||||
|
|
@ -68,6 +69,20 @@ import { createAgentCategoryMethods, type AgentCategoryMethods } from './agentCa
|
|||
/* Agent API Keys */
|
||||
import { createAgentApiKeyMethods, type AgentApiKeyMethods } from './agentApiKey';
|
||||
|
||||
export { createChannelRunStorage, ChannelRunConflict } from './channelRun';
|
||||
export { createChannelSummaryStorage, CHANNEL_SUMMARY_MAX_PRIOR_ROWS } from './channelSummary';
|
||||
export type {
|
||||
ChannelSummaryStorage,
|
||||
ChannelSummaryScope,
|
||||
ChannelSummaryRow,
|
||||
} from './channelSummary';
|
||||
export type {
|
||||
ChannelRunStorage,
|
||||
ChannelRunEnrollment,
|
||||
ChannelLaneScope,
|
||||
ChannelRunRecord,
|
||||
ChannelLaneRecord,
|
||||
} from './channelRun';
|
||||
/* MCP Servers */
|
||||
import { createMCPServerMethods, type MCPServerMethods } from './mcpServer';
|
||||
import { createCodeEnvironmentMethods, type CodeEnvironmentMethods } from './codeEnvironment';
|
||||
|
|
@ -341,6 +356,7 @@ export type AllMethods = PermissionPrincipalStorage &
|
|||
|
||||
/** Dependencies injected from the api layer into createMethods */
|
||||
export interface CreateMethodsDeps {
|
||||
getChannelStorageConfig?: () => Promise<Partial<ChannelStorageConfig>>;
|
||||
/** Matches a model name to a canonical key. From @librechat/api. */
|
||||
matchModelName?: (model: string, endpoint?: string) => string | undefined;
|
||||
/** Finds the first key in values whose key is a substring of model. From @librechat/api. */
|
||||
|
|
@ -529,7 +545,10 @@ export function createMethods(
|
|||
};
|
||||
const agentMethods = createAgentMethods(mongoose, agentDeps);
|
||||
return {
|
||||
...createUserMethods(mongoose, { getCache: deps.getCache }),
|
||||
...createUserMethods(mongoose, {
|
||||
getCache: deps.getCache,
|
||||
getChannelStorageConfig: deps.getChannelStorageConfig,
|
||||
}),
|
||||
...createToolApprovalGrantMethods(mongoose),
|
||||
...createSessionMethods(mongoose),
|
||||
...createTokenMethods(mongoose),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import mongoose, { FilterQuery } from 'mongoose';
|
||||
import { AUTH_USER_DOC_MAX_AGE_MS } from 'librechat-data-provider';
|
||||
import type {
|
||||
ChannelStorageConfig,
|
||||
BalanceRefillMode,
|
||||
RefillIntervalUnit,
|
||||
StatefulCodeEnvironment,
|
||||
|
|
@ -16,6 +17,10 @@ import type {
|
|||
} from '~/types';
|
||||
import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types';
|
||||
import type { CacheStore } from '~/types';
|
||||
import {
|
||||
eraseChannelPersistenceForUser,
|
||||
resolveChannelErasureConfig,
|
||||
} from '~/methods/channelErasure';
|
||||
import { closeResponseScopes, eraseResponseRecords } from './responseScope';
|
||||
import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search';
|
||||
import { evictAuthUserDocsForUsers } from '~/utils/eviction';
|
||||
|
|
@ -35,6 +40,7 @@ const TWO_FACTOR_ENROLLMENT_PROJECTION =
|
|||
'+totpSecret +backupCodes +pendingTotpSecret +pendingBackupCodes +twoFactorAcknowledgementNonceHash +twoFactorFinalizationNonceHash';
|
||||
|
||||
interface UserMethodDeps {
|
||||
getChannelStorageConfig?: () => Promise<Partial<ChannelStorageConfig>>;
|
||||
getCache?: (key: string) => CacheStore | undefined;
|
||||
/** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */
|
||||
delay?: (ms: number) => Promise<void>;
|
||||
|
|
@ -644,8 +650,18 @@ export function createUserMethods(
|
|||
const User = mongoose.models.User;
|
||||
const owner = await User.findById(userId).select('tenantId').lean<Pick<IUser, 'tenantId'>>();
|
||||
if (owner) {
|
||||
const channelConfig = resolveChannelErasureConfig(
|
||||
(await deps.getChannelStorageConfig?.()) ?? {},
|
||||
);
|
||||
await closeResponseScopes(mongoose, { userId, tenantId: owner.tenantId });
|
||||
await eraseResponseRecords(mongoose, { userId, tenantId: owner.tenantId });
|
||||
// Admission writes the same owner anchor in its insertion transaction.
|
||||
// Closing it therefore precedes both this sweep and every delayed retry.
|
||||
await eraseChannelPersistenceForUser(
|
||||
mongoose,
|
||||
{ userId, tenantId: owner.tenantId },
|
||||
channelConfig,
|
||||
);
|
||||
}
|
||||
await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId });
|
||||
const result = await User.deleteOne({ _id: userId });
|
||||
|
|
|
|||
179
packages/data-schemas/src/models/channelRun.ts
Normal file
179
packages/data-schemas/src/models/channelRun.ts
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
import { Schema } from 'mongoose';
|
||||
import type {
|
||||
ChannelRunView,
|
||||
ChannelRunPrincipal,
|
||||
ChannelDeliveryReceipt,
|
||||
} from 'librechat-data-provider';
|
||||
import type { Model } from 'mongoose';
|
||||
import { applyTenantIsolation } from './plugins/tenantIsolation';
|
||||
|
||||
export interface ChannelRunRecord extends Omit<ChannelRunView, 'revision'> {
|
||||
tenantId: string;
|
||||
agentId: string;
|
||||
laneId: string;
|
||||
principal: ChannelRunPrincipal;
|
||||
payloadHash: string;
|
||||
receivedAt: number;
|
||||
leaseOwner?: string;
|
||||
leaseEpoch: number;
|
||||
leaseExpiresAt: number;
|
||||
/** Durable fairness stamp; not execution authority or a retry policy. */
|
||||
claimCheckedAt?: number;
|
||||
/** Persisted before dispatch; every retry attaches to this exact identity. */
|
||||
execution?: {
|
||||
protocol?: 'responses';
|
||||
responseId?: string;
|
||||
apiKeyId?: string;
|
||||
encryptedApiKey?: string;
|
||||
requestId: string;
|
||||
conversationId: string;
|
||||
parentMessageId: string;
|
||||
generationCreatedAt?: number;
|
||||
deliveryKey?: string;
|
||||
enqueueAttemptedAt?: number;
|
||||
};
|
||||
errorCode?: string;
|
||||
deliveredAt?: number;
|
||||
}
|
||||
|
||||
export interface ChannelLaneRecord {
|
||||
tenantId: string;
|
||||
userId: string;
|
||||
channelId: string;
|
||||
laneId: string;
|
||||
nextSequence: number;
|
||||
activeRunId?: string;
|
||||
conversationId?: string;
|
||||
responseId?: string;
|
||||
}
|
||||
|
||||
export function createChannelRunModel(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): Model<ChannelRunRecord> {
|
||||
const receipt = new Schema<ChannelDeliveryReceipt>(
|
||||
{
|
||||
operationId: { type: String, required: true },
|
||||
outcome: {
|
||||
type: String,
|
||||
required: true,
|
||||
enum: ['planned', 'inflight', 'confirmed', 'retryable', 'unknown', 'permanent_failure'],
|
||||
},
|
||||
attempts: { type: Number, required: true },
|
||||
externalMessageKey: String,
|
||||
externalFileId: String,
|
||||
retryAt: Number,
|
||||
errorCode: String,
|
||||
upload: {
|
||||
type: new Schema(
|
||||
{
|
||||
fileId: { type: String, required: true },
|
||||
stage: {
|
||||
type: String,
|
||||
required: true,
|
||||
enum: ['allocated', 'uploaded'],
|
||||
},
|
||||
},
|
||||
{ _id: false },
|
||||
),
|
||||
},
|
||||
},
|
||||
{ _id: false },
|
||||
);
|
||||
const schema = new Schema<ChannelRunRecord>(
|
||||
{
|
||||
tenantId: { type: String, required: true },
|
||||
channelId: { type: String, required: true },
|
||||
agentId: { type: String, required: true },
|
||||
laneId: { type: String, required: true },
|
||||
runId: { type: String, required: true },
|
||||
state: {
|
||||
type: String,
|
||||
required: true,
|
||||
enum: [
|
||||
'queued',
|
||||
'executing',
|
||||
'requires_action',
|
||||
'delivering',
|
||||
'delivered',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'blocked',
|
||||
],
|
||||
},
|
||||
admission: { type: Schema.Types.Mixed, required: true },
|
||||
principal: { type: Schema.Types.Mixed, required: true },
|
||||
payloadHash: { type: String, required: true },
|
||||
sequence: { type: Number, required: true },
|
||||
receivedAt: { type: Number, required: true },
|
||||
leaseOwner: String,
|
||||
leaseEpoch: { type: Number, required: true, default: 0 },
|
||||
leaseExpiresAt: { type: Number, required: true, default: 0 },
|
||||
claimCheckedAt: { type: Number, default: 0 },
|
||||
execution: Schema.Types.Mixed,
|
||||
snapshot: Schema.Types.Mixed,
|
||||
progress: Schema.Types.Mixed,
|
||||
plan: Schema.Types.Mixed,
|
||||
receipts: { type: [receipt], required: true, default: [] },
|
||||
cancellationRequested: { type: Boolean, required: true, default: false },
|
||||
errorCode: String,
|
||||
deliveredAt: Number,
|
||||
},
|
||||
{ timestamps: true, minimize: false },
|
||||
);
|
||||
schema.index({ tenantId: 1, runId: 1 }, { unique: true });
|
||||
schema.index({ tenantId: 1, 'principal.userId': 1 });
|
||||
schema.index(
|
||||
{
|
||||
tenantId: 1,
|
||||
channelId: 1,
|
||||
'admission.actor.teamId': 1,
|
||||
'admission.actor.appId': 1,
|
||||
'admission.eventId': 1,
|
||||
},
|
||||
{ unique: true, name: 'channel_run_event_v2' },
|
||||
);
|
||||
schema.index({ tenantId: 1, laneId: 1, sequence: 1 }, { unique: true });
|
||||
schema.index(
|
||||
{ tenantId: 1, laneId: 1, state: 1, sequence: 1 },
|
||||
{ name: 'channel_run_active_predecessor' },
|
||||
);
|
||||
schema.index({ tenantId: 1, channelId: 1, state: 1, leaseExpiresAt: 1, receivedAt: 1 });
|
||||
schema.index(
|
||||
{
|
||||
tenantId: 1,
|
||||
channelId: 1,
|
||||
state: 1,
|
||||
claimCheckedAt: 1,
|
||||
receivedAt: 1,
|
||||
sequence: 1,
|
||||
leaseExpiresAt: 1,
|
||||
},
|
||||
{ name: 'channel_run_fair_claim' },
|
||||
);
|
||||
schema.index({ state: 1, _id: 1 });
|
||||
schema.index({ tenantId: 1, channelId: 1, 'receipts.externalMessageKey': 1 });
|
||||
applyTenantIsolation(schema);
|
||||
return mongoose.models.ChannelRun ?? mongoose.model<ChannelRunRecord>('ChannelRun', schema);
|
||||
}
|
||||
|
||||
export function createChannelLaneModel(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): Model<ChannelLaneRecord> {
|
||||
const schema = new Schema<ChannelLaneRecord>(
|
||||
{
|
||||
tenantId: { type: String, required: true },
|
||||
userId: { type: String, required: true },
|
||||
channelId: { type: String, required: true },
|
||||
laneId: { type: String, required: true },
|
||||
nextSequence: { type: Number, required: true, default: 0 },
|
||||
activeRunId: String,
|
||||
conversationId: String,
|
||||
responseId: String,
|
||||
},
|
||||
{ timestamps: true },
|
||||
);
|
||||
schema.index({ tenantId: 1, laneId: 1 }, { unique: true });
|
||||
schema.index({ tenantId: 1, userId: 1 });
|
||||
applyTenantIsolation(schema);
|
||||
return mongoose.models.ChannelLane ?? mongoose.model<ChannelLaneRecord>('ChannelLane', schema);
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { createAgentQueuedTurnModel, createAgentQueuedTurnSequenceModel } from './queuedTurn';
|
||||
|
||||
import { createChannelRunModel, createChannelLaneModel } from './channelRun';
|
||||
import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';
|
||||
import { createScheduleModel, createScheduleRunModel } from './schedule';
|
||||
import { getTenantIndexMigrationHint } from '~/migrations/tenantIndexes';
|
||||
|
|
@ -75,6 +76,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
|
|||
SubagentContent: ReturnType<typeof createSubagentContentModel>;
|
||||
Agent: ReturnType<typeof createAgentModel>;
|
||||
AgentApiKey: ReturnType<typeof createAgentApiKeyModel>;
|
||||
ChannelRun: ReturnType<typeof createChannelRunModel>;
|
||||
ChannelLane: ReturnType<typeof createChannelLaneModel>;
|
||||
AgentCategory: ReturnType<typeof createAgentCategoryModel>;
|
||||
MCPServer: ReturnType<typeof createMCPServerModel>;
|
||||
Role: ReturnType<typeof createRoleModel>;
|
||||
|
|
@ -132,6 +135,8 @@ export function createModels(mongoose: typeof import('mongoose')): {
|
|||
SubagentContent: createSubagentContentModel(mongoose),
|
||||
Agent: createAgentModel(mongoose),
|
||||
AgentApiKey: createAgentApiKeyModel(mongoose),
|
||||
ChannelRun: createChannelRunModel(mongoose),
|
||||
ChannelLane: createChannelLaneModel(mongoose),
|
||||
AgentCategory: createAgentCategoryModel(mongoose),
|
||||
MCPServer: createMCPServerModel(mongoose),
|
||||
Role: createRoleModel(mongoose),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { Schema } from 'mongoose';
|
||||
import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider';
|
||||
import { applySearchTokens } from '~/models/plugins/searchTokens';
|
||||
import { isReservedTenantId } from '~/config/tenantContext';
|
||||
import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search';
|
||||
import { IUser } from '~/types';
|
||||
|
||||
|
|
@ -232,6 +233,10 @@ const userSchema: Schema<IUser> = new Schema<IUser>(
|
|||
tenantId: {
|
||||
type: String,
|
||||
index: true,
|
||||
validate: {
|
||||
validator: (value: string | undefined) => !isReservedTenantId(value),
|
||||
message: 'Reserved tenant identifier',
|
||||
},
|
||||
},
|
||||
},
|
||||
{ timestamps: true },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue