mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 07:23:11 +00:00
🗂️ perf: Index Hot Lookups and Fix the Message Cursor (#17464)
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Vote Gate / Gate (push) Waiting to run
Codegraph E2E Votes / vote (shard 1/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 10/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 2/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 3/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 4/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 5/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 6/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 7/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 8/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 9/10) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/chat (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
Some checks are pending
Backend Unit Tests / Build packages (push) Waiting to run
Backend Unit Tests / Codegraph select (push) Waiting to run
Backend Unit Tests / TypeScript type checks (push) Blocked by required conditions
Backend Unit Tests / Circular dependency checks (push) Waiting to run
Backend Unit Tests / Tests: api (shard 1/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 2/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: api (shard 3/3) (push) Blocked by required conditions
Backend Unit Tests / Tests: data-provider (push) Blocked by required conditions
Backend Unit Tests / Tests: data-schemas (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 1/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 2/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 3/4) (push) Blocked by required conditions
Backend Unit Tests / Tests: @librechat/api (shard 4/4) (push) Blocked by required conditions
Codegraph E2E Vote Gate / Gate (push) Waiting to run
Codegraph E2E Votes / vote (shard 1/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 10/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 2/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 3/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 4/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 5/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 6/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 7/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 8/10) (push) Waiting to run
Codegraph E2E Votes / vote (shard 9/10) (push) Waiting to run
Docker Dev Branch Images Build / publish (push) Waiting to run
Frontend Unit Tests / Client build recovery regression (push) Waiting to run
Frontend Unit Tests / Codegraph select (push) Waiting to run
Frontend Unit Tests / Build packages (push) Waiting to run
Frontend Unit Tests / TypeScript type checks (client) (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/chat (push) Blocked by required conditions
Frontend Unit Tests / Tests: @librechat/client (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 1/2) (push) Blocked by required conditions
Frontend Unit Tests / Tests: Ubuntu (shard 2/2) (push) Blocked by required conditions
Frontend Unit Tests / Vite build verification (push) Blocked by required conditions
* perf: Index Hot Lookups and Make Message Cursor Pagination Stable Add narrow indexes for session refresh-token deletes, user-only tool call deletes, queued-turn delivery and reconciliation claims, agent code environment references, and updatedAt message pages. Add a TTL index to the KeyvMongo collection. getMessagesByCursor now sorts and pages on a unique (sortField, _id) tiebreaker with a millisecond-precise cursor while still accepting legacy cursors, and GET /api/messages caps pageSize in a TypeScript helper. * fix: Keep the Message Id for the Cursor Under the Client Projection * fix: Declare select on the Message Cursor Method Type * feat: Make the Message Page Size Ceiling Configurable * fix: Clamp the Default Message Page Size to the Configured Ceiling * test: Cover Message Cursor Paging and Page Size Cap End to End * fix: Continue Message Cursors Whose Sort Value Is Empty * fix: Include Missing Sort Values When Paging Messages Descending * perf: Resolve the Message Page Ceiling Beside the Ownership Read * fix: Observe the Ownership Read While the Page Size Resolves * style: Sort Imports in the Message Page Module * refactor: Move Page Size Read Observation Into the API Package * fix: Keep Missing and Empty Sort Values Distinct in the Message Cursor * fix: Page Past Messages With a Missing Timestamp
This commit is contained in:
parent
6515ec1387
commit
c81e244afd
20 changed files with 781 additions and 12 deletions
|
|
@ -59,6 +59,12 @@ jest.mock('@librechat/api', () => {
|
|||
/** The real helper, without loading the rest of the package this suite mocks around. */
|
||||
withoutTraceRefs: jest.requireActual('../../../../packages/api/src/langfuse/trace.ts')
|
||||
.withoutTraceRefs,
|
||||
/** Mirrors the real parse (default 25, ceiling 1000) without loading the config service. */
|
||||
awaitPageSize: jest.fn((pageSize) => pageSize),
|
||||
resolveMessagesPageSize: jest.fn(async (_user, raw) => {
|
||||
const parsed = parseInt(String(raw), 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.min(parsed, 1000) : 25;
|
||||
}),
|
||||
createPrivateTextView: jest.fn(() => (_req, _res, next) => next()),
|
||||
stripPrivateMessageFields: jest.requireActual(
|
||||
'../../../../packages/api/src/protection/private/view',
|
||||
|
|
@ -613,7 +619,7 @@ describe('message route conversation ownership filters', () => {
|
|||
name: 'capped',
|
||||
query: `search=needle&pageSize=${MEILI_SEARCH_LIMIT + 1}`,
|
||||
searchLimit: MEILI_SEARCH_LIMIT,
|
||||
conversationLimit: MEILI_SEARCH_LIMIT + 1,
|
||||
conversationLimit: MEILI_SEARCH_LIMIT,
|
||||
},
|
||||
{
|
||||
name: 'invalid',
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ const {
|
|||
createToolCallPartHandler,
|
||||
createSubagentAttachmentCallsHandler,
|
||||
rejectToolCallPreviewWrites,
|
||||
resolveMessagesPageSize,
|
||||
awaitPageSize,
|
||||
withMessageToolCallPreviews,
|
||||
} = require('@librechat/api');
|
||||
const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore');
|
||||
|
|
@ -132,8 +134,8 @@ router.get('/', async (req, res) => {
|
|||
messageId,
|
||||
search,
|
||||
} = req.query;
|
||||
const parsedPageSize = parseInt(pageSizeRaw, 10);
|
||||
const pageSize = Number.isFinite(parsedPageSize) && parsedPageSize > 0 ? parsedPageSize : 25;
|
||||
/** Starts beside the ownership read; it never rejects and falls back to the default ceiling. */
|
||||
const pageSizeRead = resolveMessagesPageSize(req.user, pageSizeRaw, toolCallPreviewDeps);
|
||||
|
||||
let response;
|
||||
const sortField = ['endpoint', 'createdAt', 'updatedAt'].includes(sortBy)
|
||||
|
|
@ -145,6 +147,7 @@ router.get('/', async (req, res) => {
|
|||
const previewToolCalls = prepareToolCallPreviews(req, toolCallPreviewDeps);
|
||||
if (typeof conversationId === 'string') {
|
||||
const ownershipRead = db.getConvoOwnership(user, conversationId);
|
||||
const pageSize = await awaitPageSize(pageSizeRead, ownershipRead);
|
||||
/** Client-facing reads never expose server-private fields such as `contextMeta`. */
|
||||
const messageRead = messageId
|
||||
? db.getMessages({ conversationId, messageId, user }, CLIENT_MESSAGE_SELECT)
|
||||
|
|
@ -189,6 +192,7 @@ router.get('/', async (req, res) => {
|
|||
messages: await previewToolCalls(messageResult.value.messages),
|
||||
};
|
||||
} else if (search) {
|
||||
const pageSize = await pageSizeRead;
|
||||
const searchResults = await db.searchMessages(
|
||||
search,
|
||||
{
|
||||
|
|
|
|||
89
e2e/specs/mock/scenarios/message-cursor.spec.ts
Normal file
89
e2e/specs/mock/scenarios/message-cursor.spec.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { getE2EUser } from '../../../setup/user';
|
||||
import {
|
||||
deleteConversations,
|
||||
deleteMessagesByConversation,
|
||||
seedConversations,
|
||||
withMongo,
|
||||
} from '../db';
|
||||
import { NEW_CHAT_PATH, getAccessToken, requestJson } from '../helpers';
|
||||
|
||||
type Page = { messages: { messageId: string; _id?: string }[]; nextCursor: string | null };
|
||||
|
||||
async function seedTied(conversationId: string, count: number): Promise<string[]> {
|
||||
const at = new Date('2026-01-01T00:00:00.123Z');
|
||||
await seedConversations(getE2EUser().email, [{ conversationId, title: 'cursor', updatedAt: at }]);
|
||||
return withMongo(async (db) => {
|
||||
const user = await db.collection('users').findOne({ email: getE2EUser().email });
|
||||
const rows = Array.from({ length: count }, (_, index) => ({
|
||||
messageId: `cursor-${index}-${randomUUID()}`,
|
||||
conversationId,
|
||||
user: String(user?._id),
|
||||
text: `message ${index}`,
|
||||
sender: 'User',
|
||||
isCreatedByUser: false,
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
__v: 0,
|
||||
}));
|
||||
await db.collection('messages').insertMany(rows);
|
||||
return rows.map((row) => row.messageId);
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('message cursor and page size', () => {
|
||||
const conversationIds: string[] = [];
|
||||
test.afterEach(async () => {
|
||||
await deleteMessagesByConversation(conversationIds);
|
||||
await deleteConversations(conversationIds);
|
||||
conversationIds.length = 0;
|
||||
});
|
||||
|
||||
test('@scenario:message-pages-share-timestamp-without-skips messages sharing one timestamp are all returned across pages', async ({
|
||||
page,
|
||||
}) => {
|
||||
const conversationId = randomUUID();
|
||||
conversationIds.push(conversationId);
|
||||
const seeded = await seedTied(conversationId, 7);
|
||||
await page.goto(NEW_CHAT_PATH);
|
||||
const token = await getAccessToken(page);
|
||||
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let index = 0; index < 10; index++) {
|
||||
const query = new URLSearchParams({ conversationId, pageSize: '3' });
|
||||
if (cursor) {
|
||||
query.set('cursor', cursor);
|
||||
}
|
||||
const result: Page = await requestJson<Page>(page, {
|
||||
path: `/api/messages?${query.toString()}`,
|
||||
token,
|
||||
});
|
||||
expect(result.messages.every((message) => message._id === undefined)).toBe(true);
|
||||
seen.push(...result.messages.map((message) => message.messageId));
|
||||
cursor = result.nextCursor;
|
||||
if (!cursor) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
expect(seen).toHaveLength(7);
|
||||
expect(new Set(seen)).toEqual(new Set(seeded));
|
||||
});
|
||||
|
||||
test('@scenario:message-page-size-capped an oversized pageSize returns a bounded page', async ({
|
||||
page,
|
||||
}) => {
|
||||
const conversationId = randomUUID();
|
||||
conversationIds.push(conversationId);
|
||||
await seedTied(conversationId, 4);
|
||||
await page.goto(NEW_CHAT_PATH);
|
||||
const token = await getAccessToken(page);
|
||||
const result = await requestJson<Page>(page, {
|
||||
path: `/api/messages?conversationId=${conversationId}&pageSize=999999999`,
|
||||
token,
|
||||
});
|
||||
expect(result.messages).toHaveLength(4);
|
||||
expect(result.nextCursor).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -571,6 +571,10 @@ registration:
|
|||
# violationLog:
|
||||
# maxEntries: 100
|
||||
|
||||
# Largest pageSize accepted when reading a conversation's messages.
|
||||
# messages:
|
||||
# maxPageSize: 1000 # default: 1000
|
||||
|
||||
# rateLimits:
|
||||
# # Per-user MCP App browser-route limits. Each limit uses a fixed one-minute window.
|
||||
# mcpApps:
|
||||
|
|
|
|||
49
packages/api/src/cache/keyvMongo.spec.ts
vendored
Normal file
49
packages/api/src/cache/keyvMongo.spec.ts
vendored
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import keyvMongo from './keyvMongo';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
...jest.requireActual('@librechat/data-schemas'),
|
||||
logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
|
||||
let server: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await MongoMemoryServer.create();
|
||||
await mongoose.connect(server.getUri());
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await server?.stop();
|
||||
});
|
||||
|
||||
async function waitForIndex(name: string): Promise<Record<string, unknown> | undefined> {
|
||||
const logs = mongoose.connection.db!.collection('logs');
|
||||
for (let attempt = 0; attempt < 50; attempt++) {
|
||||
const found = (await logs.indexes()).find((index) => index.name === name);
|
||||
if (found) {
|
||||
return found as Record<string, unknown>;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
describe('KeyvMongo logs collection', () => {
|
||||
it('indexes lookups by key and expires entries through a TTL index', async () => {
|
||||
await keyvMongo.set('ban:1', 'value', 60_000);
|
||||
expect(await keyvMongo.get('ban:1')).toBe('value');
|
||||
expect(await waitForIndex('key_1')).toBeDefined();
|
||||
const expiry = await waitForIndex('expiresAt_1');
|
||||
expect(expiry?.expireAfterSeconds).toBe(0);
|
||||
});
|
||||
|
||||
it('stores a null expiresAt for entries without a ttl, which the TTL index ignores', async () => {
|
||||
const logs = mongoose.connection.db!.collection('logs');
|
||||
await keyvMongo.set('no-ttl', 'kept');
|
||||
expect(await logs.countDocuments({ key: 'no-ttl' })).toBe(1);
|
||||
const stored = await logs.findOne({ key: 'no-ttl' });
|
||||
expect(stored?.expiresAt).toBeNull();
|
||||
});
|
||||
});
|
||||
8
packages/api/src/cache/keyvMongo.ts
vendored
8
packages/api/src/cache/keyvMongo.ts
vendored
|
|
@ -85,6 +85,14 @@ class KeyvMongoCustom extends EventEmitter {
|
|||
).catch((error: unknown) => {
|
||||
logger.warn(`[KeyvMongo] Could not index "${collection}" by key`, error);
|
||||
});
|
||||
/** `set` stamps `expiresAt` from the ttl (null when there is none, which a TTL index
|
||||
* ignores), so entries Keyv already treats as expired are also removed from disk. */
|
||||
buildIndexWithRetry(
|
||||
() => store.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }),
|
||||
`KeyvMongo "${collection}" expiry index`,
|
||||
).catch((error: unknown) => {
|
||||
logger.warn(`[KeyvMongo] Could not add expiry index to "${collection}"`, error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
export * from './parts';
|
||||
export * from './previews';
|
||||
export * from './page';
|
||||
|
|
|
|||
68
packages/api/src/messages/page.spec.ts
Normal file
68
packages/api/src/messages/page.spec.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import {
|
||||
DEFAULT_MESSAGES_PAGE_SIZE,
|
||||
MAX_MESSAGES_PAGE_SIZE,
|
||||
parseMessagesPageSize,
|
||||
resolveMessagesPageSize,
|
||||
awaitPageSize,
|
||||
} from './page';
|
||||
|
||||
describe('parseMessagesPageSize', () => {
|
||||
it('falls back to the default for missing, zero, negative, or non-numeric values', () => {
|
||||
for (const raw of [undefined, '', '0', '-5', 'abc', null, {}, ['10']]) {
|
||||
expect(parseMessagesPageSize(raw)).toBe(DEFAULT_MESSAGES_PAGE_SIZE);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps values within the cap unchanged', () => {
|
||||
expect(parseMessagesPageSize('10')).toBe(10);
|
||||
expect(parseMessagesPageSize(String(MAX_MESSAGES_PAGE_SIZE))).toBe(MAX_MESSAGES_PAGE_SIZE);
|
||||
});
|
||||
|
||||
it('clamps oversized values to the cap', () => {
|
||||
expect(parseMessagesPageSize('1000000')).toBe(MAX_MESSAGES_PAGE_SIZE);
|
||||
});
|
||||
|
||||
it('honors a configured ceiling', () => {
|
||||
expect(parseMessagesPageSize('5000', 2000)).toBe(2000);
|
||||
expect(parseMessagesPageSize('50', 2000)).toBe(50);
|
||||
expect(parseMessagesPageSize(undefined, 10)).toBe(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveMessagesPageSize', () => {
|
||||
const user = { id: 'u1', role: 'USER' };
|
||||
|
||||
it('applies the configured ceiling', async () => {
|
||||
const getAppConfig = jest.fn().mockResolvedValue({ messages: { maxPageSize: 40 } });
|
||||
await expect(resolveMessagesPageSize(user, '500', { getAppConfig })).resolves.toBe(40);
|
||||
expect(getAppConfig).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ skipRuntimeAugmentation: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps the default ceiling when no limit is configured', async () => {
|
||||
const getAppConfig = jest.fn().mockResolvedValue({});
|
||||
await expect(resolveMessagesPageSize(user, '5000', { getAppConfig })).resolves.toBe(
|
||||
MAX_MESSAGES_PAGE_SIZE,
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default ceiling when config is unavailable', async () => {
|
||||
const getAppConfig = jest.fn().mockRejectedValue(new Error('down'));
|
||||
await expect(resolveMessagesPageSize(user, '5000', { getAppConfig })).resolves.toBe(
|
||||
MAX_MESSAGES_PAGE_SIZE,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('awaitPageSize', () => {
|
||||
it('returns the page size and leaves a rejecting sibling read awaitable', async () => {
|
||||
const failing = Promise.reject(new Error('db down'));
|
||||
const size = await awaitPageSize(
|
||||
new Promise<number>((resolve) => setTimeout(() => resolve(40), 5)),
|
||||
failing,
|
||||
);
|
||||
expect(size).toBe(40);
|
||||
await expect(failing).rejects.toThrow('db down');
|
||||
});
|
||||
});
|
||||
57
packages/api/src/messages/page.ts
Normal file
57
packages/api/src/messages/page.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import type { AppConfigUserLike } from '~/app/service';
|
||||
import type { ToolCallPreviewDeps } from './previews';
|
||||
import { getAppConfigOptionsFromUser } from '~/app/service';
|
||||
|
||||
export const DEFAULT_MESSAGES_PAGE_SIZE = 25;
|
||||
/** Default ceiling, matching the search page ceiling; `messages.maxPageSize` overrides it. */
|
||||
export const MAX_MESSAGES_PAGE_SIZE = 1000;
|
||||
|
||||
/** Parses the client `pageSize` query value, falling back to the default and never exceeding the cap. */
|
||||
export function parseMessagesPageSize(
|
||||
raw: unknown,
|
||||
maxPageSize: number = MAX_MESSAGES_PAGE_SIZE,
|
||||
): number {
|
||||
const parsed =
|
||||
typeof raw === 'string' || typeof raw === 'number' ? parseInt(String(raw), 10) : NaN;
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
return Math.min(DEFAULT_MESSAGES_PAGE_SIZE, maxPageSize);
|
||||
}
|
||||
return Math.min(parsed, maxPageSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the client `pageSize` against the deployment's `messages.maxPageSize`. The config read skips
|
||||
* runtime augmentation and is meant to start beside the message reads; an unavailable config keeps
|
||||
* the default ceiling instead of failing the load.
|
||||
*/
|
||||
export async function resolveMessagesPageSize(
|
||||
user: AppConfigUserLike | undefined,
|
||||
raw: unknown,
|
||||
deps: ToolCallPreviewDeps,
|
||||
): Promise<number> {
|
||||
try {
|
||||
const config = await deps.getAppConfig({
|
||||
...getAppConfigOptionsFromUser(user),
|
||||
skipRuntimeAugmentation: true,
|
||||
});
|
||||
return parseMessagesPageSize(raw, config?.messages?.maxPageSize ?? MAX_MESSAGES_PAGE_SIZE);
|
||||
} catch (error) {
|
||||
logger.warn('[messages] Config unavailable; using the default page size ceiling', error);
|
||||
return parseMessagesPageSize(raw);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for the resolved page size while `pending` reads run beside it. They are observed so one
|
||||
* that rejects first is not an unhandled rejection; each stays awaitable by the caller.
|
||||
*/
|
||||
export async function awaitPageSize(
|
||||
pageSize: Promise<number>,
|
||||
...pending: Promise<unknown>[]
|
||||
): Promise<number> {
|
||||
for (const read of pending) {
|
||||
read.catch(() => undefined);
|
||||
}
|
||||
return pageSize;
|
||||
}
|
||||
|
|
@ -4662,6 +4662,12 @@ export const configSchema = z.object({
|
|||
.optional(),
|
||||
timeouts: outboundTimeoutsSchema.optional(),
|
||||
violationLog: violationLogSchema.optional(),
|
||||
messages: z
|
||||
.object({
|
||||
/** Largest `pageSize` accepted by `GET /api/messages`. Defaults to 1000. */
|
||||
maxPageSize: z.number().int().min(1).max(10000).optional(),
|
||||
})
|
||||
.optional(),
|
||||
fileConfig: fileConfigSchema.optional(),
|
||||
modelSpecs: specsConfigSchema.optional(),
|
||||
filters: filtersConfigSchema.optional(),
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ export const AppService = async (params?: {
|
|||
githubCompare: config.githubCompare,
|
||||
timeouts: config.timeouts,
|
||||
violationLog: config.violationLog,
|
||||
messages: config.messages,
|
||||
mcpSettings,
|
||||
mcpAppSandbox,
|
||||
fileStrategy,
|
||||
|
|
|
|||
275
packages/data-schemas/src/methods/message.cursor.spec.ts
Normal file
275
packages/data-schemas/src/methods/message.cursor.spec.ts
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { createMessageMethods, CLIENT_MESSAGE_SELECT } from './message';
|
||||
import { createModels } from '../models';
|
||||
|
||||
jest.mock('~/config/winston', () => ({
|
||||
error: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
info: jest.fn(),
|
||||
debug: jest.fn(),
|
||||
}));
|
||||
|
||||
let server: MongoMemoryServer;
|
||||
const methods = createMessageMethods(mongoose);
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await MongoMemoryServer.create();
|
||||
Object.assign(mongoose.models, createModels(mongoose));
|
||||
await mongoose.connect(server.getUri());
|
||||
await mongoose.models.Message.createIndexes();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await server?.stop();
|
||||
});
|
||||
afterEach(async () => {
|
||||
await mongoose.models.Message.deleteMany({});
|
||||
});
|
||||
|
||||
async function seedTied(conversationId: string, count: number, at: Date): Promise<string[]> {
|
||||
const rows = Array.from({ length: count }, (_, index) => ({
|
||||
messageId: `m-${index}-${uuid()}`,
|
||||
conversationId,
|
||||
user: 'user-1',
|
||||
text: `message ${index}`,
|
||||
isCreatedByUser: false,
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
}));
|
||||
await mongoose.models.Message.collection.insertMany(rows);
|
||||
return rows.map((row) => row.messageId);
|
||||
}
|
||||
|
||||
async function readAll(options: {
|
||||
sortField: string;
|
||||
sortOrder: 1 | -1;
|
||||
conversationId: string;
|
||||
limit: number;
|
||||
}): Promise<string[]> {
|
||||
const ids: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId: options.conversationId, user: 'user-1' },
|
||||
{ sortField: options.sortField, sortOrder: options.sortOrder, limit: options.limit, cursor },
|
||||
);
|
||||
ids.push(...result.messages.map((message) => message.messageId));
|
||||
if (result.nextCursor == null) {
|
||||
return ids;
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
throw new Error('cursor did not terminate');
|
||||
}
|
||||
|
||||
describe('getMessagesByCursor', () => {
|
||||
it.each([
|
||||
['createdAt', -1],
|
||||
['createdAt', 1],
|
||||
['updatedAt', -1],
|
||||
['updatedAt', 1],
|
||||
] as const)(
|
||||
'pages through rows sharing one %s value without skips or repeats (order %i)',
|
||||
async (sortField, sortOrder) => {
|
||||
const conversationId = uuid();
|
||||
const seeded = await seedTied(conversationId, 7, new Date('2026-01-01T00:00:00.123Z'));
|
||||
const seen = await readAll({ sortField, sortOrder, conversationId, limit: 3 });
|
||||
expect(seen).toHaveLength(7);
|
||||
expect(new Set(seen)).toEqual(new Set(seeded));
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps millisecond precision in the cursor', async () => {
|
||||
const conversationId = uuid();
|
||||
await seedTied(conversationId, 3, new Date('2026-01-01T00:00:00.456Z'));
|
||||
const first = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ limit: 1 },
|
||||
);
|
||||
expect(first.nextCursor).toContain('2026-01-01T00:00:00.456Z|');
|
||||
});
|
||||
|
||||
it('pages ties without skips under the client projection that omits _id', async () => {
|
||||
const conversationId = uuid();
|
||||
const seeded = await seedTied(conversationId, 7, new Date('2026-01-01T00:00:00.123Z'));
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ limit: 3, cursor, select: CLIENT_MESSAGE_SELECT },
|
||||
);
|
||||
expect(result.messages.every((message) => message._id === undefined)).toBe(true);
|
||||
seen.push(...result.messages.map((message) => message.messageId));
|
||||
if (result.nextCursor == null) {
|
||||
break;
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
expect(seen).toHaveLength(7);
|
||||
expect(new Set(seen)).toEqual(new Set(seeded));
|
||||
});
|
||||
|
||||
it('pages past rows with a missing sort value without skips', async () => {
|
||||
const conversationId = uuid();
|
||||
const at = new Date('2026-01-01T00:00:00.000Z');
|
||||
const rows = Array.from({ length: 5 }, (_, index) => ({
|
||||
messageId: `blank-${index}-${uuid()}`,
|
||||
conversationId,
|
||||
user: 'user-1',
|
||||
text: 'x',
|
||||
isCreatedByUser: false,
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
}));
|
||||
await mongoose.models.Message.collection.insertMany(rows);
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < 10; page++) {
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ sortField: 'endpoint', sortOrder: 1, limit: 2, cursor },
|
||||
);
|
||||
seen.push(...result.messages.map((message) => message.messageId));
|
||||
if (result.nextCursor == null) {
|
||||
break;
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
expect(seen).toHaveLength(5);
|
||||
expect(new Set(seen)).toEqual(new Set(rows.map((row) => row.messageId)));
|
||||
});
|
||||
|
||||
it('descending past populated values still returns rows with a missing sort value', async () => {
|
||||
const conversationId = uuid();
|
||||
const at = new Date('2026-01-01T00:00:00.000Z');
|
||||
const rows = ['a', 'a', 'b', undefined, undefined].map((endpoint, index) => ({
|
||||
messageId: `mixed-${index}-${uuid()}`,
|
||||
conversationId,
|
||||
user: 'user-1',
|
||||
text: 'x',
|
||||
isCreatedByUser: false,
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
...(endpoint ? { endpoint } : {}),
|
||||
}));
|
||||
await mongoose.models.Message.collection.insertMany(rows);
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < 10; page++) {
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ sortField: 'endpoint', sortOrder: -1, limit: 2, cursor },
|
||||
);
|
||||
seen.push(...result.messages.map((message) => message.messageId));
|
||||
if (result.nextCursor == null) {
|
||||
break;
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
expect(seen).toHaveLength(5);
|
||||
expect(new Set(seen)).toEqual(new Set(rows.map((row) => row.messageId)));
|
||||
});
|
||||
|
||||
it.each([
|
||||
[1, 1],
|
||||
[1, 2],
|
||||
[-1, 1],
|
||||
[-1, 2],
|
||||
] as const)(
|
||||
'keeps missing, empty and populated sort values distinct (order %i, page size %i)',
|
||||
async (sortOrder, limit) => {
|
||||
const conversationId = uuid();
|
||||
const at = new Date('2026-01-01T00:00:00.000Z');
|
||||
const rows = [undefined, '', undefined, '', 'a', 'a', 'b', null].map((endpoint, index) => ({
|
||||
messageId: `mix-${index}-${uuid()}`,
|
||||
conversationId,
|
||||
user: 'user-1',
|
||||
text: 'x',
|
||||
isCreatedByUser: false,
|
||||
createdAt: at,
|
||||
updatedAt: at,
|
||||
...(endpoint === undefined ? {} : { endpoint }),
|
||||
}));
|
||||
await mongoose.models.Message.collection.insertMany(rows);
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ sortField: 'endpoint', sortOrder, limit, cursor },
|
||||
);
|
||||
seen.push(...result.messages.map((message) => message.messageId));
|
||||
if (result.nextCursor == null) {
|
||||
break;
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
expect(seen).toHaveLength(rows.length);
|
||||
expect(new Set(seen)).toEqual(new Set(rows.map((row) => row.messageId)));
|
||||
},
|
||||
);
|
||||
|
||||
it.each([1, -1] as const)(
|
||||
'pages past messages with a missing timestamp (order %i)',
|
||||
async (sortOrder) => {
|
||||
const conversationId = uuid();
|
||||
const at = new Date('2026-01-01T00:00:00.000Z');
|
||||
const rows = [at, undefined, at, undefined].map((updatedAt, index) => ({
|
||||
messageId: `ts-${index}-${uuid()}`,
|
||||
conversationId,
|
||||
user: 'user-1',
|
||||
text: 'x',
|
||||
isCreatedByUser: false,
|
||||
createdAt: at,
|
||||
...(updatedAt ? { updatedAt } : {}),
|
||||
}));
|
||||
await mongoose.models.Message.collection.insertMany(rows);
|
||||
const seen: string[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < 10; page++) {
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ sortField: 'updatedAt', sortOrder, limit: 1, cursor },
|
||||
);
|
||||
seen.push(...result.messages.map((message) => message.messageId));
|
||||
if (result.nextCursor == null) {
|
||||
break;
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
expect(new Set(seen)).toEqual(new Set(rows.map((row) => row.messageId)));
|
||||
expect(seen).toHaveLength(rows.length);
|
||||
},
|
||||
);
|
||||
|
||||
it('still honors a legacy bare timestamp cursor', async () => {
|
||||
const conversationId = uuid();
|
||||
const newer = new Date('2026-02-01T00:00:00.000Z');
|
||||
const older = new Date('2026-01-01T00:00:00.000Z');
|
||||
await seedTied(conversationId, 1, newer);
|
||||
const [olderId] = await seedTied(conversationId, 1, older);
|
||||
const result = await methods.getMessagesByCursor(
|
||||
{ conversationId, user: 'user-1' },
|
||||
{ cursor: newer.toISOString(), limit: 10 },
|
||||
);
|
||||
expect(result.messages.map((message) => message.messageId)).toEqual([olderId]);
|
||||
});
|
||||
|
||||
it('serves the default and updatedAt sorts from an index', async () => {
|
||||
const Message = mongoose.models.Message;
|
||||
const conversationId = uuid();
|
||||
await seedTied(conversationId, 3, new Date());
|
||||
for (const sortField of ['createdAt', 'updatedAt']) {
|
||||
const plan = (await Message.find({ conversationId, user: 'user-1' })
|
||||
.sort({ [sortField]: -1, _id: -1 })
|
||||
.explain('queryPlanner')) as unknown as { queryPlanner: { winningPlan: unknown } };
|
||||
const winning = JSON.stringify(plan.queryPlanner.winningPlan);
|
||||
expect(winning.includes('"stage":"IXSCAN"')).toBe(true);
|
||||
expect(winning.includes('"stage":"SORT"')).toBe(false);
|
||||
expect(winning.includes('COLLSCAN')).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -950,6 +950,7 @@ export interface MessageMethods {
|
|||
sortOrder?: 1 | -1;
|
||||
limit?: number;
|
||||
cursor?: string | null;
|
||||
select?: string;
|
||||
},
|
||||
): Promise<{ messages: IMessage[]; nextCursor: string | null }>;
|
||||
searchMessages(
|
||||
|
|
@ -960,6 +961,58 @@ export interface MessageMethods {
|
|||
deleteMessages(filter: FilterQuery<IMessage>): Promise<DeleteResult>;
|
||||
}
|
||||
|
||||
const DATE_SORT_FIELDS = new Set(['createdAt', 'updatedAt']);
|
||||
/** Cursor marker for a null or missing sort value, which MongoDB orders before every string, '' included. */
|
||||
const NULL_SORT_VALUE = '\u0000';
|
||||
const OBJECT_ID_PATTERN = /^[0-9a-f]{24}$/i;
|
||||
|
||||
/** Encodes the last row of a page as `<sortValue>|<_id>`; dates use ISO strings so milliseconds survive. */
|
||||
function formatMessageCursor(value: unknown, id: unknown): string {
|
||||
if (value == null) {
|
||||
return id == null ? NULL_SORT_VALUE : `${NULL_SORT_VALUE}|${String(id)}`;
|
||||
}
|
||||
const sortValue = value instanceof Date ? value.toISOString() : String(value);
|
||||
return id == null ? sortValue : `${sortValue}|${String(id)}`;
|
||||
}
|
||||
|
||||
/** Accepts both the compound cursor and the legacy bare sort value from older clients. */
|
||||
function parseMessageCursor(cursor: string): { value: string; id?: string } {
|
||||
const split = cursor.lastIndexOf('|');
|
||||
if (split >= 0 && OBJECT_ID_PATTERN.test(cursor.slice(split + 1))) {
|
||||
return { value: cursor.slice(0, split), id: cursor.slice(split + 1) };
|
||||
}
|
||||
return { value: cursor };
|
||||
}
|
||||
|
||||
/** The rows after a cursor, following MongoDB's order: null/missing, then '' (strings only), then values. */
|
||||
function buildCursorFilter(
|
||||
sortField: string,
|
||||
op: '$gt' | '$lt',
|
||||
value: string | Date,
|
||||
position: { value: string; id?: Types.ObjectId },
|
||||
): FilterQuery<IMessage> {
|
||||
if (position.id == null) {
|
||||
return { [sortField]: { [op]: value } };
|
||||
}
|
||||
const tieId = { _id: { [op]: position.id } };
|
||||
const ascending = op === '$gt';
|
||||
if (position.value === NULL_SORT_VALUE) {
|
||||
return {
|
||||
$or: [
|
||||
{ [sortField]: null, ...tieId },
|
||||
...(ascending ? [{ [sortField]: { $ne: null } }] : []),
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
$or: [
|
||||
{ [sortField]: { [op]: value } },
|
||||
{ [sortField]: value, ...tieId },
|
||||
...(ascending ? [] : [{ [sortField]: null }]),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/** The agent-ownership rule shared by background-tool settling and claiming:
|
||||
* `agentId` on the part wins, then `tool_call.agentId`, and a part without
|
||||
* agent identity belongs to any caller (single-agent runs). `field: null`
|
||||
|
|
@ -4247,16 +4300,28 @@ export function createMessageMethods(
|
|||
) {
|
||||
const Message = mongoose.models.Message as Model<IMessage>;
|
||||
const { sortField = 'createdAt', sortOrder = -1, limit = 25, cursor, select } = options;
|
||||
const queryFilter = { ...filter };
|
||||
if (cursor) {
|
||||
queryFilter[sortField] = sortOrder === 1 ? { $gt: cursor } : { $lt: cursor };
|
||||
const queryFilter: FilterQuery<IMessage> = { ...filter };
|
||||
const position = cursor ? parseMessageCursor(cursor) : null;
|
||||
if (position != null) {
|
||||
const op = sortOrder === 1 ? '$gt' : '$lt';
|
||||
const value =
|
||||
DATE_SORT_FIELDS.has(sortField) && position.value !== NULL_SORT_VALUE
|
||||
? new Date(position.value)
|
||||
: position.value;
|
||||
const id = position.id == null ? undefined : new mongoose.Types.ObjectId(position.id);
|
||||
const after = buildCursorFilter(sortField, op, value, { value: position.value, id });
|
||||
queryFilter.$and = [...(queryFilter.$and ?? []), after];
|
||||
}
|
||||
const query = Message.find(queryFilter);
|
||||
if (select) {
|
||||
query.select(select);
|
||||
/** The cursor needs `_id`, so an `-_id` projection is applied after it is read. */
|
||||
const tokens = select ? select.split(/\s+/).filter(Boolean) : [];
|
||||
const omitId = tokens.includes('-_id');
|
||||
if (tokens.length > 0) {
|
||||
query.select(tokens.filter((token) => token !== '-_id').join(' '));
|
||||
}
|
||||
/** `_id` makes the order total, so a page boundary inside a run of equal sort values is stable. */
|
||||
const messages = await query
|
||||
.sort({ [sortField]: sortOrder })
|
||||
.sort({ [sortField]: sortOrder, _id: sortOrder })
|
||||
.limit(limit + 1)
|
||||
.lean<IMessage[]>();
|
||||
|
||||
|
|
@ -4264,9 +4329,12 @@ export function createMessageMethods(
|
|||
if (messages.length > limit) {
|
||||
messages.pop();
|
||||
const last = messages[messages.length - 1];
|
||||
const cursorValue =
|
||||
sortField === 'createdAt' ? last.createdAt : last[sortField as keyof IMessage];
|
||||
nextCursor = String(cursorValue ?? '');
|
||||
nextCursor = formatMessageCursor(last[sortField as keyof IMessage], last._id);
|
||||
}
|
||||
if (omitId) {
|
||||
for (const message of messages) {
|
||||
delete (message as { _id?: unknown })._id;
|
||||
}
|
||||
}
|
||||
return { messages, nextCursor };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -201,5 +201,20 @@ agentSchema.index({ tenantId: 1, updatedAt: -1, _id: 1 });
|
|||
agentSchema.index({ createdAt: -1, _id: 1 });
|
||||
agentSchema.index({ tenantId: 1, createdAt: -1, _id: 1 });
|
||||
agentSchema.index({ 'edges.to': 1 });
|
||||
/** Reference checks before code environment removal match either field; each `$or` branch needs its own index. */
|
||||
agentSchema.index(
|
||||
{ code_environment_id: 1 },
|
||||
{
|
||||
name: 'agent_code_environment_id',
|
||||
partialFilterExpression: { code_environment_id: { $exists: true } },
|
||||
},
|
||||
);
|
||||
agentSchema.index(
|
||||
{ code_environment_ids: 1 },
|
||||
{
|
||||
name: 'agent_code_environment_ids',
|
||||
partialFilterExpression: { code_environment_ids: { $exists: true } },
|
||||
},
|
||||
);
|
||||
|
||||
export default agentSchema;
|
||||
|
|
|
|||
91
packages/data-schemas/src/schema/lookupIndexes.spec.ts
Normal file
91
packages/data-schemas/src/schema/lookupIndexes.spec.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import type { FilterQuery, Model } from 'mongoose';
|
||||
import { createModels } from '../models';
|
||||
|
||||
let server: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await MongoMemoryServer.create();
|
||||
Object.assign(mongoose.models, createModels(mongoose));
|
||||
await mongoose.connect(server.getUri());
|
||||
await Promise.all(
|
||||
['Session', 'ToolCall', 'AgentQueuedTurn', 'Agent'].map((name) =>
|
||||
mongoose.models[name].createIndexes(),
|
||||
),
|
||||
);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await server?.stop();
|
||||
});
|
||||
|
||||
/** Names the index the winning plan scans, or `COLLSCAN` when it reads the whole collection. */
|
||||
async function winningScan(
|
||||
modelName: string,
|
||||
filter: FilterQuery<unknown>,
|
||||
seed: Record<string, unknown>[],
|
||||
): Promise<string> {
|
||||
const model = mongoose.models[modelName] as Model<unknown>;
|
||||
await model.collection.deleteMany({});
|
||||
/** Filler rows without the indexed field keep the planner from preferring a scan on a tiny collection. */
|
||||
await model.collection.insertMany(seed);
|
||||
const plan = (await model.find(filter).explain('queryPlanner')) as unknown as {
|
||||
queryPlanner: { winningPlan: unknown };
|
||||
};
|
||||
const text = JSON.stringify(plan.queryPlanner.winningPlan);
|
||||
const match = /"stage":"IXSCAN".*?"indexName":"([^"]+)"/.exec(text);
|
||||
return match?.[1] ?? (text.includes('COLLSCAN') ? 'COLLSCAN' : text);
|
||||
}
|
||||
|
||||
/** Builds rows that satisfy each collection's unique indexes via `make(index)`. */
|
||||
const filler = (count: number, make: (index: number) => Record<string, unknown>) =>
|
||||
Array.from({ length: count }, (_, index) => ({ filler: index, ...make(index) }));
|
||||
|
||||
describe('single-field lookups use an index', () => {
|
||||
it('session by refreshTokenHash without a user', async () => {
|
||||
const scan = await winningScan('Session', { refreshTokenHash: 'abc' }, [
|
||||
...filler(50, (i) => ({
|
||||
user: new mongoose.Types.ObjectId(),
|
||||
refreshTokenHash: `other-${i}`,
|
||||
})),
|
||||
{ user: new mongoose.Types.ObjectId(), refreshTokenHash: 'abc' },
|
||||
]);
|
||||
expect(scan).toBe('refreshTokenHash_1');
|
||||
});
|
||||
|
||||
it('tool calls by user alone', async () => {
|
||||
const user = new mongoose.Types.ObjectId();
|
||||
const scan = await winningScan('ToolCall', { user }, [
|
||||
...filler(50, () => ({ user: new mongoose.Types.ObjectId() })),
|
||||
{ user },
|
||||
]);
|
||||
expect(scan).toBe('user_1');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['deliveryKey', { deliveryKey: 'key-1' }, 'agent_queued_turn_delivery_key'],
|
||||
[
|
||||
'reconciliationClaimId',
|
||||
{ reconciliationClaimId: 'claim-1', reconciliationClaimBy: 'worker' },
|
||||
'agent_queued_turn_reconciliation_claim',
|
||||
],
|
||||
])('queued turns by %s', async (_name, filter, indexName) => {
|
||||
const scan = await winningScan('AgentQueuedTurn', filter, [
|
||||
...filler(50, (i) => ({ clientRequestId: `filler-${i}` })),
|
||||
{ ...filter, clientRequestId: 'target', filler: -1 },
|
||||
]);
|
||||
expect(scan).toBe(indexName);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['code_environment_id', 'agent_code_environment_id', 'env-1'],
|
||||
['code_environment_ids', 'agent_code_environment_ids', 'env-1'],
|
||||
])('agents by %s', async (field, indexName, value) => {
|
||||
const scan = await winningScan('Agent', { [field]: value }, [
|
||||
...filler(50, (i) => ({ id: `filler-${i}` })),
|
||||
{ id: 'target', [field]: field.endsWith('s') ? [value] : value, filler: -1 },
|
||||
]);
|
||||
expect(scan).toBe(indexName);
|
||||
});
|
||||
});
|
||||
|
|
@ -365,6 +365,12 @@ messageSchema.index({
|
|||
*/
|
||||
messageSchema.index({ conversationId: 1, user: 1, createdAt: 1, _id: 1 });
|
||||
|
||||
/**
|
||||
* Serves the cursor-paginated message read when sorted by `updatedAt`, the
|
||||
* default of `GET /api/messages`; the `createdAt` sort uses the index above.
|
||||
*/
|
||||
messageSchema.index({ conversationId: 1, user: 1, updatedAt: 1, _id: 1 });
|
||||
|
||||
/**
|
||||
* Serves the batched newest-task read for child threads. `user` is an equality
|
||||
* prefix and `conversationId` is the partition key, so Mongo/DocumentDB can
|
||||
|
|
|
|||
|
|
@ -191,6 +191,22 @@ queuedTurnSchema.index(
|
|||
},
|
||||
{ name: 'agent_queued_turn_active' },
|
||||
);
|
||||
/** Delivery retirement looks turns up by `deliveryKey` alone; most turns never carry one. */
|
||||
queuedTurnSchema.index(
|
||||
{ deliveryKey: 1 },
|
||||
{
|
||||
name: 'agent_queued_turn_delivery_key',
|
||||
partialFilterExpression: { deliveryKey: { $exists: true } },
|
||||
},
|
||||
);
|
||||
/** Re-fetches the rows a reconciliation sweep just claimed. */
|
||||
queuedTurnSchema.index(
|
||||
{ reconciliationClaimId: 1 },
|
||||
{
|
||||
name: 'agent_queued_turn_reconciliation_claim',
|
||||
partialFilterExpression: { reconciliationClaimId: { $exists: true } },
|
||||
},
|
||||
);
|
||||
queuedTurnSchema.index({ tenantId: 1, user: 1, status: 1, claimUntil: 1 });
|
||||
queuedTurnSchema.index({ status: 1, createdAt: 1, _id: 1 });
|
||||
queuedTurnSchema.index({
|
||||
|
|
|
|||
|
|
@ -23,5 +23,7 @@ const sessionSchema: Schema<ISession> = new Schema({
|
|||
});
|
||||
|
||||
sessionSchema.index({ user: 1, refreshTokenHash: 1 }, { unique: true });
|
||||
/** `deleteSession({ refreshToken })` has no user in its filter, so the compound index above cannot serve it. */
|
||||
sessionSchema.index({ refreshTokenHash: 1 });
|
||||
|
||||
export default sessionSchema;
|
||||
|
|
|
|||
|
|
@ -59,6 +59,8 @@ const toolCallSchema: Schema<IToolCallData> = new Schema(
|
|||
);
|
||||
|
||||
toolCallSchema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });
|
||||
/** Serves the user-only `deleteToolCalls` (account and bulk cleanup) without a collection scan. */
|
||||
toolCallSchema.index({ user: 1 });
|
||||
toolCallSchema.index({ messageId: 1, user: 1, tenantId: 1 });
|
||||
toolCallSchema.index({ conversationId: 1, user: 1, tenantId: 1 });
|
||||
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ export interface AppConfig {
|
|||
timeouts?: TCustomConfig['timeouts'];
|
||||
/** Violation log retention; unset keeps the server default */
|
||||
violationLog?: TCustomConfig['violationLog'];
|
||||
messages?: TCustomConfig['messages'];
|
||||
skillDeletion?: {
|
||||
finalizationEnabled?: boolean;
|
||||
sweepIntervalMs?: number;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue