From de9d46fcd4f45feff849a848a84ff01becf619cb Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:41:37 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=8E=20perf:=20Index-Bound=20Word-Prefi?= =?UTF-8?q?x=20Search=20for=20Users=20and=20Groups=20(#16719)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: Debounce People Picker Search * perf: Index-Bound Word-Prefix Search for Users and Groups Principal search and admin user search matched name, email and username with case-insensitive regexes, which scan the whole users collection. Users and groups now store lowercase, accent-folded token arrays derived from each searchable field, indexed with tenantId, and searches match anchored token prefixes. Schema middleware keeps the tokens in sync on save, insertMany and update queries. Documents written before the tokens existed stay findable through a fallback branch until npm run migrate:search-tokens backfills them; startup warns while any remain. * perf: Build Search Token Indexes in the Backfill Migration * fix: Reconcile Stale Search Tokens and Close Write-Path Gaps The backfill now compares every document's tokens with its source fields and repairs stale ones, so writes from an older server during a rolling deploy are recoverable by rerunning it. Undefined assignments no longer clear tokens, upsert inserts initialize every token field, every query word is required, relevance scoring folds accents like the filter does, and the startup check counts both collections in parallel and owns its error logging. * perf: Drop the Second People Picker Debounce SearchPicker already debounces query changes by 500ms; the extra 300ms delay in UnifiedPeopleSearch stacked on top of it. * fix: Match Nothing for Queries Over the Word Cap Dropping the query word cap let one request build an unbounded filter. Restore a 16 word cap, but a longer query now matches nothing instead of ignoring its extra words, so a cap never produces a false positive. * test: Cover Word-Prefix People Search and the Token Backfill End to End * fix: Tokenize Fields an Upsert Copies From Its Filter MongoDB copies equality conditions from an upsert's filter into the inserted document, so their token fields now derive from that value instead of being initialized empty. * fix: Rank Accent-Folded User Matches Like the Search Filter * test: Run the Token Backfill Without Blocking the Playwright Worker * perf: Index Search Tokens Tenant First for Scoped Searches and Ignore Undefined Unsets * test: Skip the Module-Scope Index Rebuild When the Scenario Runs the Backfill * fix: Exit the Search Token Migration When Redis Clients Stay Open * perf: Probe for Missing Search Tokens at Startup Instead of Counting Them * test: Explain the Startup Probe the Warning Actually Issues * fix: Keep Search Token Fields Out of Serialized Users and Groups * fix: Keep the Admin Search Legacy Fallback Anchored at the Field Start * fix: Score Groups Only on the Fields Search Matches * perf: Start the Search Token Probe Alongside the Permission Checks * fix: Warn at Startup When the Search Token Indexes Were Never Built * fix: Score People Search Matches With the Filter's Word Separators * perf: Check the Token Indexes Before Probing for Unmigrated Documents * test: Drive the Failed Startup Probe Past the Index Check * fix: Score People Search Matches Independently of Word Order --- .../start/__tests__/migration.spec.js | 37 ++ api/server/services/start/migration.js | 5 +- config/migrate-search-tokens.js | 30 ++ e2e/specs/mock/db.ts | 2 +- .../scenarios/people-search-tokens.spec.ts | 226 ++++++++++++ package.json | 2 + packages/api/src/admin/users.spec.ts | 39 ++- packages/api/src/admin/users.ts | 17 +- packages/data-schemas/src/index.ts | 3 + .../src/methods/user.methods.spec.ts | 35 +- packages/data-schemas/src/methods/user.ts | 52 +-- .../src/methods/userGroup.spec.ts | 72 +++- .../data-schemas/src/methods/userGroup.ts | 64 +--- packages/data-schemas/src/migrations/index.ts | 2 + .../src/migrations/searchTokens.spec.ts | 247 +++++++++++++ .../src/migrations/searchTokens.ts | 178 ++++++++++ .../src/models/plugins/searchTokens.spec.ts | 318 +++++++++++++++++ .../src/models/plugins/searchTokens.ts | 71 ++++ packages/data-schemas/src/schema/group.ts | 3 + packages/data-schemas/src/schema/user.ts | 3 + packages/data-schemas/src/types/group.ts | 3 + packages/data-schemas/src/types/user.ts | 4 + packages/data-schemas/src/utils/index.ts | 1 + .../data-schemas/src/utils/search.spec.ts | 215 ++++++++++++ packages/data-schemas/src/utils/search.ts | 328 ++++++++++++++++++ 25 files changed, 1834 insertions(+), 123 deletions(-) create mode 100644 api/server/services/start/__tests__/migration.spec.js create mode 100644 config/migrate-search-tokens.js create mode 100644 e2e/specs/mock/scenarios/people-search-tokens.spec.ts create mode 100644 packages/data-schemas/src/migrations/searchTokens.spec.ts create mode 100644 packages/data-schemas/src/migrations/searchTokens.ts create mode 100644 packages/data-schemas/src/models/plugins/searchTokens.spec.ts create mode 100644 packages/data-schemas/src/models/plugins/searchTokens.ts create mode 100644 packages/data-schemas/src/utils/search.spec.ts create mode 100644 packages/data-schemas/src/utils/search.ts diff --git a/api/server/services/start/__tests__/migration.spec.js b/api/server/services/start/__tests__/migration.spec.js new file mode 100644 index 0000000000..ba4bfc32d8 --- /dev/null +++ b/api/server/services/start/__tests__/migration.spec.js @@ -0,0 +1,37 @@ +const mockWarnOnMissingSearchTokens = jest.fn(); +const mockCheckAgentPermissionsMigration = jest.fn(); +const mockCheckPromptPermissionsMigration = jest.fn(); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { error: jest.fn() }, + warnOnMissingSearchTokens: (...args) => mockWarnOnMissingSearchTokens(...args), +})); +jest.mock('@librechat/api', () => ({ + logAgentMigrationWarning: jest.fn(), + logPromptMigrationWarning: jest.fn(), + checkAgentPermissionsMigration: (...args) => mockCheckAgentPermissionsMigration(...args), + checkPromptPermissionsMigration: (...args) => mockCheckPromptPermissionsMigration(...args), +})); +jest.mock('~/models', () => ({ findRoleByIdentifier: jest.fn() })); + +const { checkMigrations } = require('../migration'); + +describe('checkMigrations', () => { + it('starts the search token probe while the permission checks are still running', async () => { + let finishAgentCheck; + mockCheckAgentPermissionsMigration.mockReturnValue( + new Promise((resolve) => { + finishAgentCheck = resolve; + }), + ); + mockCheckPromptPermissionsMigration.mockResolvedValue({}); + mockWarnOnMissingSearchTokens.mockResolvedValue(undefined); + + const done = checkMigrations(); + expect(mockWarnOnMissingSearchTokens).toHaveBeenCalledTimes(1); + + finishAgentCheck({}); + await done; + expect(mockCheckPromptPermissionsMigration).toHaveBeenCalledTimes(1); + }); +}); diff --git a/api/server/services/start/migration.js b/api/server/services/start/migration.js index 70f8300e08..d170590168 100644 --- a/api/server/services/start/migration.js +++ b/api/server/services/start/migration.js @@ -1,5 +1,5 @@ const mongoose = require('mongoose'); -const { logger } = require('@librechat/data-schemas'); +const { logger, warnOnMissingSearchTokens } = require('@librechat/data-schemas'); const { logAgentMigrationWarning, logPromptMigrationWarning, @@ -13,6 +13,8 @@ const { findRoleByIdentifier } = require('~/models'); * This runs at the end to ensure all systems are initialized */ async function checkMigrations() { + /** Started first so it overlaps the permission checks; it logs its own failures. */ + const searchTokens = warnOnMissingSearchTokens(mongoose.connection); try { const agentMigrationResult = await checkAgentPermissionsMigration({ mongoose, @@ -37,6 +39,7 @@ async function checkMigrations() { } catch (error) { logger.error('Failed to check prompt permissions migration:', error); } + await searchTokens; } module.exports = { diff --git a/config/migrate-search-tokens.js b/config/migrate-search-tokens.js new file mode 100644 index 0000000000..812e0e0338 --- /dev/null +++ b/config/migrate-search-tokens.js @@ -0,0 +1,30 @@ +require('dotenv').config(); +const mongoose = require('mongoose'); +const { backfillSearchTokens } = require('@librechat/data-schemas'); +const connect = require('./connect'); + +/** + * Backfills the word-prefix search tokens on users and groups saved before they existed. + * Idempotent; safe to run while the server is up. + * + * Usage: npm run migrate:search-tokens [-- --dry-run] [-- --batch-size=500] + */ +(async () => { + let exitCode = 0; + try { + await connect(); + const batchArg = process.argv.find((arg) => arg.startsWith('--batch-size=')); + const result = await backfillSearchTokens(mongoose.connection, { + dryRun: process.argv.includes('--dry-run'), + batchSize: batchArg ? parseInt(batchArg.split('=')[1], 10) || undefined : undefined, + }); + console.log(JSON.stringify(result, null, 2)); + } catch (error) { + console.error('Search token migration failed:', error); + exitCode = 1; + } finally { + await mongoose.disconnect(); + } + /** Exit explicitly: with Redis enabled, the cache clients opened on connect keep the process alive. */ + process.exit(exitCode); +})(); diff --git a/e2e/specs/mock/db.ts b/e2e/specs/mock/db.ts index 40127f7135..4a0492eb7a 100644 --- a/e2e/specs/mock/db.ts +++ b/e2e/specs/mock/db.ts @@ -16,7 +16,7 @@ function getRuntimeEnvPath(): string { ); } -function getMongoUri(): string { +export function getMongoUri(): string { try { const env = JSON.parse(fs.readFileSync(getRuntimeEnvPath(), 'utf8')) as { MONGO_URI?: string }; if (env.MONGO_URI) { diff --git a/e2e/specs/mock/scenarios/people-search-tokens.spec.ts b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts new file mode 100644 index 0000000000..5bef523e2b --- /dev/null +++ b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts @@ -0,0 +1,226 @@ +import path from 'path'; +import { promisify } from 'util'; +import { execFile } from 'child_process'; +import { expect, test } from '@playwright/test'; +import type { Page, Locator } from '@playwright/test'; +import { MOCK_ENDPOINTS, NEW_CHAT_PATH, getAccessToken, requestJson, uniqueName } from '../helpers'; +import { openAgentBuilder, uniqueAgentName, cleanupAgent } from '../agents.helpers'; +import { withMongo, getMongoUri } from '../db'; + +/** + * People search (`GET /api/permissions/search-principals`, the share dialog's + * picker) matches word prefixes over the derived `nameTokens` / `emailTokens` / + * `usernameTokens` arrays. These scenarios register users through the real + * auth route, so the schema middleware writes the tokens, and drive the + * backfill script (`npm run migrate:search-tokens`) against this run's database + * for documents a writer left without tokens or with stale ones. + * + * The primary e2e user is the first one registered in the run, so it is ADMIN + * and may search principals and share agents without seeding role permissions + * (see `principal-type-labels.spec.ts`). + */ + +const SEARCH_LABEL = 'Search for people or groups by name or email'; +const SEARCH_PATH = '/api/permissions/search-principals'; +const REPO_ROOT = path.resolve(__dirname, '../../../..'); + +type SearchResult = { name?: string; email?: string; type: string }; + +/** Letters only, so a suffix stays one search word. */ +function suffix(): string { + return Array.from({ length: 7 }, () => + String.fromCharCode(97 + Math.floor(Math.random() * 26)), + ).join(''); +} + +async function register(page: Page, name: string, email: string, username: string) { + const password = 'Search-tokens-pass-1'; + const response = await page.request.post('/api/auth/register', { + data: { name, email, username, password, confirm_password: password }, + }); + expect(response.ok(), await response.text()).toBeTruthy(); +} + +async function search(page: Page, token: string, q: string): Promise { + const body = await requestJson<{ results: SearchResult[] }>(page, { + path: `${SEARCH_PATH}?q=${encodeURIComponent(q)}&limit=20`, + token, + }); + return body.results; +} + +async function deleteUsers(emails: string[]): Promise { + await withMongo(async (db) => { + await db.collection('users').deleteMany({ email: { $in: emails } }); + }); +} + +async function openShareDialog(page: Page): Promise<{ dialog: Locator; agentId: string }> { + const agentName = uniqueAgentName('E2E People Search'); + const token = await getAccessToken(page); + const agent = await requestJson<{ id: string }>(page, { + path: '/api/agents', + token, + method: 'POST', + body: { name: agentName, provider: MOCK_ENDPOINTS[0].label, model: MOCK_ENDPOINTS[0].model }, + }); + const form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name: agentName, exact: true }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(agentName); + await page.getByRole('button', { name: `Share ${agentName}` }).click(); + const dialog = page.getByRole('dialog', { name: `Share ${agentName}` }); + await expect(dialog).toBeVisible(); + return { dialog, agentId: agent.id }; +} + +const execFileAsync = promisify(execFile); + +/** Async so a stuck script fails the test on its timeout instead of blocking the worker. */ +async function runBackfill(): Promise { + try { + await execFileAsync('node', ['config/migrate-search-tokens.js'], { + cwd: REPO_ROOT, + env: { ...process.env, MONGO_URI: getMongoUri() }, + timeout: 60000, + killSignal: 'SIGKILL', + }); + } catch (error) { + const { stdout = '', stderr = '' } = error as { stdout?: string; stderr?: string }; + throw new Error( + `Backfill failed: ${String(error)}\n${stdout.slice(-2000)}\n${stderr.slice(-2000)}`, + ); + } +} + +test.describe('people search over word-prefix tokens', () => { + test('@scenario:people-search-matches-word-prefixes the share dialog finds a user by the start of each name word, and not by a fragment inside a word', async ({ + page, + }) => { + test.setTimeout(120000); + const id = suffix(); + const user = { name: `Quill${id} Brightwater`, email: `${uniqueName('quill')}@search.test` }; + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await register(page, user.name, user.email, `quill${id}`); + + const opened = await openShareDialog(page); + agentId = opened.agentId; + const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL }); + await searchBox.fill(`brig quill${id.slice(0, 3)}`); + await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({ + timeout: 10000, + }); + + const token = await getAccessToken(page); + expect((await search(page, token, `ill${id}`)).map((r) => r.email)).not.toContain(user.email); + expect((await search(page, token, user.email.slice(0, 12))).map((r) => r.email)).toContain( + user.email, + ); + } finally { + await cleanupAgent(page, agentId); + await deleteUsers([user.email]); + } + }); + + test('@scenario:people-search-folds-accents-and-ranks-exact-first an unaccented query finds an accented name and ranks its exact match above a longer prefix match', async ({ + page, + }) => { + const id = suffix(); + const exact = { name: `Zoë${id}`, email: `${uniqueName('zoe-exact')}@search.test` }; + const prefix = { name: `Zoe${id}x Park`, email: `${uniqueName('zoe-prefix')}@search.test` }; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await register(page, prefix.name, prefix.email, `zoep${id}`); + await register(page, exact.name, exact.email, `zoee${id}`); + + const token = await getAccessToken(page); + const results = await search(page, token, `zoe${id}`); + expect(results.map((r) => r.name)).toEqual([exact.name, prefix.name]); + } finally { + await deleteUsers([exact.email, prefix.email]); + } + }); + + test('@scenario:backfill-repairs-missing-and-stale-tokens a user saved without tokens is still found, a user whose tokens went stale is found under the new name once the backfill runs', async ({ + page, + }) => { + test.setTimeout(120000); + const id = suffix(); + const legacy = { name: `Legacy${id} Person`, email: `${uniqueName('legacy')}@search.test` }; + const renamed = { name: `Renamed${id} Person`, email: `${uniqueName('renamed')}@search.test` }; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + const primary = await withMongo((db) => + db.collection('users').findOne({}, { sort: { createdAt: 1 } }), + ); + /** Raw writes, as an older server would leave them: one user with no + * tokens at all, one renamed without its tokens being rewritten. */ + await withMongo(async (db) => { + const now = new Date(); + await db.collection('users').insertMany([ + { ...legacy, tenantId: primary?.tenantId, createdAt: now, updatedAt: now }, + { + ...renamed, + tenantId: primary?.tenantId, + nameTokens: [`original${id}`, 'person'], + emailTokens: [renamed.email], + usernameTokens: [], + createdAt: now, + updatedAt: now, + }, + ]); + }); + + const token = await getAccessToken(page); + const emails = async (q: string) => (await search(page, token, q)).map((r) => r.email); + expect(await emails(`gacy${id}`)).toContain(legacy.email); + expect(await emails(`renamed${id}`)).not.toContain(renamed.email); + + await runBackfill(); + + expect(await emails(`renamed${id}`)).toContain(renamed.email); + expect(await emails(`original${id}`)).not.toContain(renamed.email); + expect(await emails(`legacy${id}`)).toContain(legacy.email); + const stored = await withMongo((db) => + db.collection('users').findOne({ email: legacy.email }), + ); + expect(stored?.nameTokens).toEqual([`legacy${id}`, 'person']); + } finally { + await deleteUsers([legacy.email, renamed.email]); + } + }); + + test('@scenario:people-picker-searches-once-per-pause typing a query character by character sends one search request, not one per keystroke', async ({ + page, + }) => { + test.setTimeout(120000); + const id = suffix(); + const user = { name: `Pause${id} Typist`, email: `${uniqueName('pause')}@search.test` }; + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await register(page, user.name, user.email, `pause${id}`); + const opened = await openShareDialog(page); + agentId = opened.agentId; + + const queries: string[] = []; + page.on('request', (request) => { + const url = new URL(request.url()); + if (url.pathname === SEARCH_PATH) { + queries.push(url.searchParams.get('q') ?? ''); + } + }); + const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL }); + await searchBox.pressSequentially(`pause${id}`, { delay: 40 }); + await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({ + timeout: 10000, + }); + expect(queries).toEqual([`pause${id}`]); + } finally { + await cleanupAgent(page, agentId); + await deleteUsers([user.email]); + } + }); +}); diff --git a/package.json b/package.json index 468915b789..72dde5ff7f 100644 --- a/package.json +++ b/package.json @@ -143,6 +143,8 @@ "migrate:tenant-indexes": "node config/migrate-tenant-indexes.js", "migrate:tenant-indexes:dry-run": "node config/migrate-tenant-indexes.js --dry-run", "migrate:terms-timestamp": "node config/migrate-terms-timestamp.js", + "migrate:search-tokens": "node config/migrate-search-tokens.js", + "migrate:search-tokens:dry-run": "node config/migrate-search-tokens.js --dry-run", "flush-cache": "node config/flush-cache.js", "migrate:agent-permissions:dry-run": "node config/migrate-agent-permissions.js --dry-run", "migrate:agent-permissions": "node config/migrate-agent-permissions.js", diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index 3c1aea1303..b3582c33f1 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -1,4 +1,5 @@ import { Types } from 'mongoose'; +import { buildUserSearchFilter } from '@librechat/data-schemas'; import { PrincipalType, SystemRoles } from 'librechat-data-provider'; import type { IUser, UserDeleteResult } from '@librechat/data-schemas'; import type { Response } from 'express'; @@ -205,20 +206,23 @@ describe('createAdminUsersHandlers', () => { expect(response.capped).toBe(true); }); - it('searches name, email, and username with anchored prefix regex', async () => { + it('searches the indexed word-prefix tokens of name, email, and username', async () => { const findUsers = jest.fn().mockResolvedValue([]); const deps = createDeps({ findUsers }); const handlers = createAdminUsersHandlers(deps); - const { req, res } = createReqRes({ query: { q: 'test' } }); + const { req, res } = createReqRes({ query: { q: 'Test' } }); await handlers.searchUsers(req, res); const filter = findUsers.mock.calls[0][0]; - expect(filter.$or).toHaveLength(3); - expect(filter.$or[0]).toHaveProperty('name'); - expect(filter.$or[1]).toHaveProperty('email'); - expect(filter.$or[2]).toHaveProperty('username'); - expect(filter.$or[0].name.source).toBe('^test'); + expect(filter).toEqual(buildUserSearchFilter('Test', { legacyPrefix: true })); + expect(filter.$or).toEqual( + expect.arrayContaining([ + { nameTokens: /^test/ }, + { emailTokens: /^test/ }, + { usernameTokens: /^test/ }, + ]), + ); }); it('projects username in the field selection', async () => { @@ -242,8 +246,25 @@ describe('createAdminUsersHandlers', () => { await handlers.searchUsers(req, res); const filter = findUsers.mock.calls[0][0]; - expect(filter.$or[0].name).toBeInstanceOf(RegExp); - expect(filter.$or[0].name.source).toBe('^test\\.user\\+1'); + expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^test\.user\+1/ }); + /** Documents awaiting the backfill keep the old anchored prefix match. */ + expect(filter.$and[0].$or).toContainEqual({ + nameTokens: { $exists: false }, + name: /^test\.user\+1/i, + }); + }); + + it('returns no users without querying when the query has nothing to match', async () => { + const findUsers = jest.fn(); + const deps = createDeps({ findUsers }); + const handlers = createAdminUsersHandlers(deps); + const { req, res, status, json } = createReqRes({ query: { q: '\u0301\u0301' } }); + + await handlers.searchUsers(req, res); + + expect(findUsers).not.toHaveBeenCalled(); + expect(status).toHaveBeenCalledWith(200); + expect(json).toHaveBeenCalledWith({ users: [], total: 0, capped: false }); }); it('returns 400 when query is missing', async () => { diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts index ae8ac19c9c..d28569e50a 100644 --- a/packages/api/src/admin/users.ts +++ b/packages/api/src/admin/users.ts @@ -1,7 +1,7 @@ import { Types } from 'mongoose'; import { randomUUID } from 'node:crypto'; import { PrincipalType, SystemRoles } from 'librechat-data-provider'; -import { logger, isValidObjectIdString } from '@librechat/data-schemas'; +import { logger, isValidObjectIdString, buildUserSearchFilter } from '@librechat/data-schemas'; import type { IUser, IConfig, @@ -197,14 +197,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { } const searchLimit = Math.min(Math.max(1, parseInt(limitStr, 10) || 20), 50); - const escaped = trimmed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const regex = new RegExp(`^${escaped}`, 'i'); - - const users = await findUsers( - { $or: [{ name: regex }, { email: regex }, { username: regex }] }, - '_id name email username avatar', - { limit: searchLimit, sort: { name: 1 } }, - ); + const filter = buildUserSearchFilter(trimmed, { legacyPrefix: true }); + const users = filter + ? await findUsers(filter, '_id name email username avatar', { + limit: searchLimit, + sort: { name: 1 }, + }) + : []; const results: AdminUserSearchResult[] = users.map((u) => ({ id: u._id?.toString() ?? '', diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 560d21ebdb..38b269be70 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -117,4 +117,7 @@ export { migrateTenantIndexes, dropSupersededPromptGroupIndexes, backfillMCPServerNormalizedNames, + backfillSearchTokens, + warnOnMissingSearchTokens, } from './migrations'; +export type { SearchTokenBackfillResult } from './migrations'; diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index d9318173d1..7bffe5e381 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -1842,6 +1842,35 @@ describe('User Methods - Database Tests', () => { expect(results.length).toBeGreaterThan(0); }); + test('should rank an accent-folded exact match above a prefix match', async () => { + await User.create([ + { name: 'Zoe Adams', email: 'zadams@test.com', username: 'zadams', provider: 'local' }, + { name: 'Zoë', email: 'zoe-umlaut@test.com', username: 'zumlaut', provider: 'local' }, + ]); + + const results = await methods.searchUsers({ searchPattern: 'zoe', limit: 1 }); + + expect(results).toHaveLength(1); + expect((results[0] as unknown as t.IUser).name).toBe('Zoë'); + }); + + test('should rank a reordered full-name match above a longer prefix match', async () => { + await User.create([ + { + name: 'Smith Johnson', + email: 'sjohnson@test.com', + username: 'sjohnson', + provider: 'local', + }, + { name: 'John Smith', email: 'jsmith@test.com', username: 'jsmith', provider: 'local' }, + ]); + + const results = await methods.searchUsers({ searchPattern: 'smith john', limit: 1 }); + + expect(results).toHaveLength(1); + expect((results[0] as unknown as t.IUser).name).toBe('John Smith'); + }); + test('should respect limit', async () => { const results = await methods.searchUsers({ searchPattern: 'example', limit: 2 }); @@ -1869,9 +1898,11 @@ describe('User Methods - Database Tests', () => { }); const results = await methods.searchUsers({ searchPattern: '.*' }); + expect(results).toEqual([]); - expect(results).toHaveLength(1); - expect((results[0] as unknown as t.IUser).name).toBe('Literal .* User'); + const literal = await methods.searchUsers({ searchPattern: 'literal' }); + expect(literal).toHaveLength(1); + expect((literal[0] as unknown as t.IUser).name).toBe('Literal .* User'); }); test('should handle invalid regex syntax as literal search text', async () => { diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index cc95028344..94aaddb6f1 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -16,8 +16,8 @@ import type { } from '~/types'; import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types'; import type { CacheStore } from '~/types'; +import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search'; import { evictAuthUserDocs } from '~/utils/eviction'; -import { escapeRegExp } from '~/utils/string'; import { signPayload } from '~/crypto'; import logger from '~/config/winston'; @@ -891,7 +891,7 @@ export function createUserMethods( } /** - * Search for users by pattern matching on name, email, or username (case-insensitive) + * Search for users by word prefix of name, email, or username (see `buildUserSearchFilter`) * @param searchPattern - The pattern to search for * @param limit - Maximum number of results to return * @param fieldsToSelect - The fields to include or exclude in the returned documents @@ -975,12 +975,13 @@ export function createUserMethods( } const trimmedPattern = searchPattern.trim(); - const regex = new RegExp(escapeRegExp(trimmedPattern), 'i'); + const filter = buildUserSearchFilter(trimmedPattern); + if (!filter) { + return []; + } const User = mongoose.models.User; - const query = User.find({ - $or: [{ email: regex }, { name: regex }, { username: regex }], - }).limit(limit * 2); // Get more results to allow for relevance sorting + const query = User.find(filter).limit(limit * 2); // Get more results to allow for relevance sorting if (fieldsToSelect) { query.select(fieldsToSelect); @@ -988,41 +989,10 @@ export function createUserMethods( const users = await query.lean(); - // Score results by relevance - const startsWithPattern = trimmedPattern.toLowerCase(); - - const scoredUsers = users.map((user) => { - const searchableFields = [user.name, user.email, user.username].filter( - (field): field is string => typeof field === 'string' && field.length > 0, - ); - let maxScore = 0; - - for (const field of searchableFields) { - const fieldLower = field.toLowerCase(); - let score = 0; - - // Exact match gets highest score - if (fieldLower === startsWithPattern) { - score = 100; - } - // Starts with query gets high score - else if (fieldLower.startsWith(startsWithPattern)) { - score = 80; - } - // Contains query gets medium score - else if (fieldLower.includes(startsWithPattern)) { - score = 50; - } - // Default score for database match - else { - score = 10; - } - - maxScore = Math.max(maxScore, score); - } - - return { ...user, _searchScore: maxScore }; - }); + const scoredUsers = users.map((user) => ({ + ...user, + _searchScore: scoreSearchMatch([user.name, user.email, user.username], trimmedPattern), + })); /** Top results sorted by relevance */ return scoredUsers diff --git a/packages/data-schemas/src/methods/userGroup.spec.ts b/packages/data-schemas/src/methods/userGroup.spec.ts index db3212e596..9c4a3a8ce0 100644 --- a/packages/data-schemas/src/methods/userGroup.spec.ts +++ b/packages/data-schemas/src/methods/userGroup.spec.ts @@ -117,16 +117,18 @@ describe('userGroup methods', () => { expect(results[0].name).toBe('Design'); }); - it('matches on description field', async () => { + it('does not match on the description field (search covers name and email)', async () => { const results = await methods.findGroupsByNamePattern('Eng team'); - expect(results).toHaveLength(1); - expect(results[0].name).toBe('Engineering'); + expect(results).toEqual([]); }); it('treats regex metacharacters as literal text', async () => { const results = await methods.findGroupsByNamePattern('.*'); - expect(results).toHaveLength(1); - expect(results[0].name).toBe('Literal .* Group'); + expect(results).toEqual([]); + + const literal = await methods.findGroupsByNamePattern('literal'); + expect(literal).toHaveLength(1); + expect(literal[0].name).toBe('Literal .* Group'); }); it('filters by source when provided', async () => { @@ -1245,6 +1247,14 @@ describe('userGroup methods', () => { expect(score).toBe(50); }); + it('scores accent-folded matches like their plain spelling', () => { + const score = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'Zoë', source: 'local' }, + 'zoe', + ); + expect(score).toBe(100); + }); + it('returns 10 (default) when no substring or exact match', () => { const score = methods.calculateRelevanceScore( { type: PrincipalType.USER, name: 'bob', source: 'local' }, @@ -1267,17 +1277,51 @@ describe('userGroup methods', () => { expect(score).toBe(100); }); - it('checks description for GROUP type', () => { - const score = methods.calculateRelevanceScore( + it('ignores description for GROUP type, which search no longer matches', () => { + const weak = methods.calculateRelevanceScore( { type: PrincipalType.GROUP, - name: 'other', - description: 'alice team', + name: 'Corporate Platform', + description: 'plat', source: 'local', }, - 'alice', + 'plat', ); - expect(score).toBe(80); + const strong = methods.calculateRelevanceScore( + { type: PrincipalType.GROUP, name: 'Platform Team', source: 'local' }, + 'plat', + ); + expect(weak).toBe(50); + expect(strong).toBeGreaterThan(weak); + }); + + it('scores a name equal to the query up to separators as an exact match', () => { + const score = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'Mary-Jane', source: 'local' }, + 'mary jane', + ); + expect(score).toBe(100); + }); + + it('scores a reordered full-name query as exact, above a longer prefix match', () => { + const exact = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'John Smith', source: 'local' }, + 'smith john', + ); + const prefix = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'Smith Johnson', source: 'local' }, + 'smith john', + ); + expect(exact).toBe(100); + expect(prefix).toBe(80); + }); + + it('scores query words matched across fields like a contains match', () => { + const score = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'John Doe', email: 'jd@gmail.com', source: 'local' }, + 'john gmail', + ); + expect(score).toBe(50); }); it('picks the highest score across multiple fields', () => { @@ -1402,9 +1446,11 @@ describe('userGroup methods', () => { }); const results = await methods.searchPrincipals('.*', 10, [PrincipalType.USER]); + expect(results).toEqual([]); - expect(results).toHaveLength(1); - expect(results[0].name).toBe('Literal .* User'); + const literal = await methods.searchPrincipals('literal', 10, [PrincipalType.USER]); + expect(literal).toHaveLength(1); + expect(literal[0].name).toBe('Literal .* User'); }); it('handles invalid regex syntax as literal search text', async () => { diff --git a/packages/data-schemas/src/methods/userGroup.ts b/packages/data-schemas/src/methods/userGroup.ts index 451f8e97c4..feefa83460 100644 --- a/packages/data-schemas/src/methods/userGroup.ts +++ b/packages/data-schemas/src/methods/userGroup.ts @@ -4,6 +4,7 @@ import { CacheKeys, PrincipalType, SystemRoles } from 'librechat-data-provider'; import type { TPrincipalSearchResult } from 'librechat-data-provider'; import type { Model, ClientSession, FilterQuery } from 'mongoose'; import type { CacheStore, IGroup, IRole, IUser } from '~/types'; +import { buildUserSearchFilter, scoreSearchMatch, buildGroupSearchFilter } from '~/utils/search'; import { isValidObjectIdString } from '~/utils/objectId'; import { scopedCacheKey } from '~/config/tenantContext'; import { escapeRegExp } from '~/utils/string'; @@ -607,7 +608,7 @@ export function createUserGroupMethods( } /** - * Find groups by name pattern (case-insensitive partial match) + * Find groups by word prefix of name or email (see `buildGroupSearchFilter`) * @param namePattern - The name pattern to search for * @param source - Optional source filter ('entra', 'local', or null for all) * @param limit - Maximum number of results to return @@ -621,10 +622,11 @@ export function createUserGroupMethods( session?: ClientSession, ): Promise { const Group = mongoose.models.Group as Model; - const regex = new RegExp(escapeRegExp(namePattern), 'i'); - const query: Record = { - $or: [{ name: regex }, { email: regex }, { description: regex }], - }; + const filter = buildGroupSearchFilter(namePattern); + if (!filter) { + return []; + } + const query: Record = { ...filter }; if (source) { query.source = source; @@ -869,39 +871,12 @@ export function createUserGroupMethods( * @returns Relevance score (0-100) */ function calculateRelevanceScore(item: TPrincipalSearchResult, searchPattern: string): number { - const normalizedPattern = searchPattern.toLowerCase(); - - /** Get searchable text based on type */ - const searchableFields = + /** The fields each search matches on; a group's description is not one of them */ + const values = item.type === PrincipalType.USER - ? [item.name, item.email, item.username].filter(Boolean) - : [item.name, item.email, item.description].filter(Boolean); - - let maxScore = 0; - - for (const field of searchableFields) { - if (!field) continue; - const fieldLower = field.toLowerCase(); - let score = 0; - - /** Exact match gets highest score */ - if (fieldLower === normalizedPattern) { - score = 100; - } else if (fieldLower.startsWith(normalizedPattern)) { - /** Starts with query gets high score */ - score = 80; - } else if (fieldLower.includes(normalizedPattern)) { - /** Contains query gets medium score */ - score = 50; - } else { - /** Default score for database match */ - score = 10; - } - - maxScore = Math.max(maxScore, score); - } - - return maxScore; + ? [item.name, item.email, item.username] + : [item.name, item.email]; + return scoreSearchMatch(values, searchPattern); } /** @@ -970,7 +945,8 @@ export function createUserGroupMethods( } /** - * Search for principals (users and groups) by pattern matching on name/email + * Search for principals: users and groups by word prefix (see `buildSearchTokenFilter`), + * roles by name * Returns combined results in TPrincipalSearchResult format without sorting * @param searchPattern - The pattern to search for * @param limitPerType - Maximum number of results to return @@ -992,17 +968,11 @@ export function createUserGroupMethods( const escapedPattern = escapeRegExp(trimmedPattern); const promises: Promise[] = []; - if (!typeFilter || typeFilter.includes(PrincipalType.USER)) { - /** Note: searchUsers is imported from ~/models and needs to be passed in or implemented */ + const userFilter = buildUserSearchFilter(trimmedPattern); + if (userFilter && (!typeFilter || typeFilter.includes(PrincipalType.USER))) { const userFields = 'name email username avatar provider idOnTheSource role'; - /** For now, we'll use a direct query instead of searchUsers */ const User = mongoose.models.User as Model; - const regex = new RegExp(escapedPattern, 'i'); - const userQuery = User.find({ - $or: [{ name: regex }, { email: regex }, { username: regex }], - }) - .select(userFields) - .limit(limitPerType); + const userQuery = User.find(userFilter).select(userFields).limit(limitPerType); if (session) { userQuery.session(session); diff --git a/packages/data-schemas/src/migrations/index.ts b/packages/data-schemas/src/migrations/index.ts index c3ba619181..ed2699d8a4 100644 --- a/packages/data-schemas/src/migrations/index.ts +++ b/packages/data-schemas/src/migrations/index.ts @@ -2,3 +2,5 @@ export { dropSupersededTenantIndexes, migrateTenantIndexes } from './tenantIndex export { dropSupersededPromptGroupIndexes } from './promptGroupIndexes'; export { createMCPAuthorityLookupIndexes } from './mcpAuthorityIndexes'; export { MCPServerNameMigrationError, backfillMCPServerNormalizedNames } from './mcpServerNames'; +export { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens'; +export type { SearchTokenBackfillResult } from './searchTokens'; diff --git a/packages/data-schemas/src/migrations/searchTokens.spec.ts b/packages/data-schemas/src/migrations/searchTokens.spec.ts new file mode 100644 index 0000000000..38c62f81f5 --- /dev/null +++ b/packages/data-schemas/src/migrations/searchTokens.spec.ts @@ -0,0 +1,247 @@ +import mongoose from 'mongoose'; +import { Collection } from 'mongodb'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { Document, FindOptions } from 'mongodb'; +import { + searchTokenIndexes, + USER_SEARCH_TOKEN_FIELDS, + GROUP_SEARCH_TOKEN_FIELDS, +} from '~/utils/search'; +import { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens'; +import logger from '~/config/winston'; + +jest.mock('~/config/winston', () => ({ + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), +})); + +let mongoServer: MongoMemoryServer; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(async () => { + await mongoose.connection.dropDatabase(); + jest.clearAllMocks(); +}); + +const users = () => mongoose.connection.db!.collection('users'); +const groups = () => mongoose.connection.db!.collection('groups'); + +describe('backfillSearchTokens', () => { + it('backfills users and groups across tenants in batches and is idempotent', async () => { + await users().insertMany([ + { name: 'Ana Lima', email: 'ana@x.io', username: 'ana', tenantId: 'tenant-a' }, + { name: 'Ben Ode', email: 'ben@y.io', tenantId: 'tenant-b' }, + { email: 'noname@z.io' }, + { + name: 'Already Done', + email: 'done@z.io', + nameTokens: ['already', 'done'], + emailTokens: ['done@z.io', 'done', 'z', 'io'], + usernameTokens: [], + }, + ]); + await groups().insertMany([{ name: 'Core Team', source: 'local' }]); + + const result = await backfillSearchTokens(mongoose.connection, { batchSize: 2 }); + expect(result).toEqual({ + pending: { users: 3, groups: 1 }, + updated: { users: 3, groups: 1 }, + }); + + const ana = await users().findOne({ email: 'ana@x.io' }); + expect(ana).toMatchObject({ + tenantId: 'tenant-a', + nameTokens: ['ana', 'lima'], + emailTokens: ['ana@x.io', 'ana', 'x', 'io'], + usernameTokens: ['ana'], + }); + expect(await users().findOne({ email: 'noname@z.io' })).toMatchObject({ + nameTokens: [], + usernameTokens: [], + }); + expect(await groups().findOne({ name: 'Core Team' })).toMatchObject({ + nameTokens: ['core', 'team'], + emailTokens: [], + }); + + await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({ + pending: { users: 0, groups: 0 }, + updated: { users: 0, groups: 0 }, + }); + }); + + it('completes documents that a partial update tokenized for only one field', async () => { + await users().insertOne({ name: 'Part Ial', email: 'p@x.io', emailTokens: ['p@x.io'] }); + await backfillSearchTokens(mongoose.connection); + expect(await users().findOne({ email: 'p@x.io' })).toMatchObject({ + nameTokens: ['part', 'ial'], + emailTokens: ['p@x.io', 'p', 'x', 'io'], + }); + }); + + it('repairs tokens left stale by a writer that did not maintain them', async () => { + await users().insertOne({ + name: 'New Name', + email: 'stale@x.io', + nameTokens: ['old', 'name'], + emailTokens: ['stale@x.io', 'stale', 'x', 'io'], + usernameTokens: [], + }); + await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({ + pending: { users: 1, groups: 0 }, + updated: { users: 0, groups: 0 }, + }); + await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({ + pending: { users: 1, groups: 0 }, + updated: { users: 1, groups: 0 }, + }); + expect(await users().findOne({ email: 'stale@x.io' })).toMatchObject({ + nameTokens: ['new', 'name'], + }); + }); + + it('creates the token indexes when the schema indexes were never built', async () => { + await users().insertOne({ name: 'No Index', email: 'noindex@x.io' }); + await groups().insertOne({ name: 'Plain Group' }); + await backfillSearchTokens(mongoose.connection); + const userIndexes = (await users().indexes()).map((index) => index.name); + const groupIndexes = (await groups().indexes()).map((index) => index.name); + expect(userIndexes).toEqual( + expect.arrayContaining([ + 'nameTokens_1_tenantId_1', + 'emailTokens_1_tenantId_1', + 'usernameTokens_1_tenantId_1', + 'tenantId_1_nameTokens_1', + 'tenantId_1_emailTokens_1', + 'tenantId_1_usernameTokens_1', + ]), + ); + expect(groupIndexes).toEqual( + expect.arrayContaining([ + 'nameTokens_1_tenantId_1', + 'emailTokens_1_tenantId_1', + 'tenantId_1_nameTokens_1', + 'tenantId_1_emailTokens_1', + ]), + ); + }); + + it('writes nothing on a dry run', async () => { + await users().insertOne({ name: 'Dry Run', email: 'dry@x.io' }); + await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({ + pending: { users: 1, groups: 0 }, + updated: { users: 0, groups: 0 }, + }); + expect(await users().findOne({ email: 'dry@x.io' })).not.toHaveProperty('nameTokens'); + }); +}); + +describe('warnOnMissingSearchTokens', () => { + it('warns only while documents lack tokens', async () => { + await users().insertOne({ name: 'Old User', email: 'old@x.io' }); + await groups().insertOne({ name: 'Old Group' }); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users and groups')); + + jest.clearAllMocks(); + await backfillSearchTokens(mongoose.connection); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + /** Runs the startup check and explains the `users` probe it actually issued. */ + const explainUserProbe = async () => { + const findOne = jest.spyOn(Collection.prototype, 'findOne'); + try { + await warnOnMissingSearchTokens(mongoose.connection); + const call = findOne.mock.contexts.findIndex( + (collection) => collection.collectionName === 'users', + ); + const [filter, options] = findOne.mock.calls[call] as [Document, FindOptions]; + expect(options.maxTimeMS).toBeGreaterThan(0); + return await users().find(filter).limit(1).explain('executionStats'); + } finally { + findOne.mockRestore(); + } + }; + + it('warns without probing documents while the token indexes do not exist', async () => { + await users().insertMany( + Array.from({ length: 200 }, (_, i) => ({ name: `Old ${i}`, email: `old${i}@x.io` })), + ); + const findOne = jest.spyOn(Collection.prototype, 'findOne'); + try { + await warnOnMissingSearchTokens(mongoose.connection); + const probed = findOne.mock.contexts.some( + (collection) => collection.collectionName === 'users', + ); + expect(probed).toBe(false); + } finally { + findOne.mockRestore(); + } + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users')); + }); + + it('examines no documents once the backfill has built the indexes and filled every token', async () => { + await users().insertMany( + Array.from({ length: 50 }, (_, i) => ({ name: `User ${i}`, email: `u${i}@x.io` })), + ); + await backfillSearchTokens(mongoose.connection); + jest.clearAllMocks(); + const explain = await explainUserProbe(); + expect(JSON.stringify(explain.queryPlanner.winningPlan)).not.toContain('COLLSCAN'); + expect(explain.executionStats.totalDocsExamined).toBe(0); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('warns when every document has tokens but the token indexes were never built', async () => { + await users().insertOne({ + name: 'Fresh User', + email: 'fresh@x.io', + nameTokens: ['fresh', 'user'], + emailTokens: ['fresh@x.io', 'fresh', 'x', 'io'], + usernameTokens: [], + }); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users')); + + jest.clearAllMocks(); + await backfillSearchTokens(mongoose.connection); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('stays quiet on a database with no users or groups yet', async () => { + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('logs a failed probe instead of failing startup', async () => { + /** Every token index exists, so the check reaches the probe, which then fails. */ + const indexes = [...USER_SEARCH_TOKEN_FIELDS, ...GROUP_SEARCH_TOKEN_FIELDS] + .flatMap(searchTokenIndexes) + .map((key) => ({ key })); + const findOne = jest.fn(() => Promise.reject(new Error('down'))); + const broken = { + db: { collection: () => ({ indexes: async () => indexes, findOne }) }, + }; + await expect( + warnOnMissingSearchTokens(broken as unknown as typeof mongoose.connection), + ).resolves.toBeUndefined(); + expect(findOne).toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/data-schemas/src/migrations/searchTokens.ts b/packages/data-schemas/src/migrations/searchTokens.ts new file mode 100644 index 0000000000..42d668883a --- /dev/null +++ b/packages/data-schemas/src/migrations/searchTokens.ts @@ -0,0 +1,178 @@ +import type { AnyBulkWriteOperation, Document } from 'mongodb'; +import type { Connection } from 'mongoose'; +import type { SearchTokenField } from '~/utils/search'; +import { + searchTokenIndexes, + computeSearchTokenSet, + USER_SEARCH_TOKEN_FIELDS, + GROUP_SEARCH_TOKEN_FIELDS, +} from '~/utils/search'; +import { buildIndexWithRetry } from '~/utils/retry'; +import logger from '~/config/winston'; + +const DEFAULT_BATCH_SIZE = 500; +/** Upper bound on the startup probe, in case it runs before the token indexes exist. */ +const STARTUP_PROBE_MAX_TIME_MS = 5_000; + +const SEARCH_TOKEN_COLLECTIONS: ReadonlyArray<{ + name: string; + fields: readonly SearchTokenField[]; +}> = [ + { name: 'users', fields: USER_SEARCH_TOKEN_FIELDS }, + { name: 'groups', fields: GROUP_SEARCH_TOKEN_FIELDS }, +]; + +export interface SearchTokenBackfillResult { + /** Documents whose token fields are missing or no longer match their source, per collection. */ + pending: Record; + /** Documents written, per collection; zero on a dry run. */ + updated: Record; +} + +/** Index-bound: each branch seeks one token index on its missing-key bounds. */ +function missingTokens(fields: readonly SearchTokenField[]): Document { + return { $or: fields.map((field) => ({ [field.tokens]: { $exists: false } })) }; +} + +function sameTokens(stored: unknown, expected: string[]): boolean { + return ( + Array.isArray(stored) && + stored.length === expected.length && + stored.every((token, index) => token === expected[index]) + ); +} + +/** + * Brings the search-token arrays on users and groups in line with their source + * fields: fills documents saved before the tokens existed and repairs tokens + * left stale by a writer that did not maintain them (an older server during a + * rolling deploy, a raw collection write). Every document is read and only the + * ones whose tokens differ are written, so it is idempotent and resumable; run + * it again once a rollout completes. Each write is guarded by the source values + * it was computed from, so a concurrent rename (whose own update already wrote + * fresh tokens) is skipped. Runs across all tenants on the raw collections; the + * `_id` filter keeps every write on its own document, and `tenantId` is never + * written. + */ +export async function backfillSearchTokens( + connection: Connection, + options: { batchSize?: number; dryRun?: boolean } = {}, +): Promise { + const batchSize = Math.max(1, options.batchSize ?? DEFAULT_BATCH_SIZE); + const result: SearchTokenBackfillResult = { pending: {}, updated: {} }; + + for (const { name, fields } of SEARCH_TOKEN_COLLECTIONS) { + const collection = connection.db!.collection(name); + if (!options.dryRun) { + /** Deployments running with `MONGO_AUTO_INDEX` off never build schema indexes, and + * without these the token filters would scan the collection. Same specs and default + * names as the schema declaration, so this is a no-op where Mongoose already built them. */ + for (const field of fields) { + for (const index of searchTokenIndexes(field)) { + await buildIndexWithRetry( + () => collection.createIndex(index), + `createIndex(${name}.${Object.keys(index).join('_')})`, + ); + } + } + } + result.pending[name] = 0; + result.updated[name] = 0; + + const projection = Object.fromEntries( + fields.flatMap((field) => [ + [field.source, 1], + [field.tokens, 1], + ]), + ); + let batch: AnyBulkWriteOperation[] = []; + const flush = async () => { + if (batch.length === 0) { + return; + } + // eslint-disable-next-line no-restricted-syntax -- offline all-tenant migration; `_id` filters keep each write on its own document + const written = await collection.bulkWrite(batch, { ordered: false }); + result.updated[name] += written.modifiedCount; + batch = []; + }; + + for await (const doc of collection.find({}, { projection })) { + const expected = computeSearchTokenSet(fields, doc); + if (fields.every((field) => sameTokens(doc[field.tokens], expected[field.tokens]))) { + continue; + } + result.pending[name] += 1; + if (options.dryRun) { + continue; + } + const guard = Object.fromEntries( + fields.map((field) => [field.source, doc[field.source] ?? null]), + ); + batch.push({ updateOne: { filter: { _id: doc._id, ...guard }, update: { $set: expected } } }); + if (batch.length >= batchSize) { + await flush(); + } + } + await flush(); + if (!options.dryRun) { + logger.info( + `[SearchTokenMigration] ${name}: ${result.updated[name]} of ${result.pending[name]} documents updated`, + ); + } + } + return result; +} + +/** + * Logs a startup warning when users or groups still lack search tokens. Those + * documents stay findable through the slower unindexed fallback until + * `npm run migrate:search-tokens` runs. A best-effort diagnostic: a failed + * check is logged and never blocks startup. + * + * Also warns when the token indexes are missing: with `MONGO_AUTO_INDEX` off, + * a database whose documents all received tokens from the schema middleware + * still never builds them until the backfill runs. + * + * The indexes are checked first, so the probe for a token-less document only + * runs once they exist and it can seek them; `maxTimeMS` bounds it regardless. + */ +export async function warnOnMissingSearchTokens(connection: Connection): Promise { + try { + const pending = await Promise.all( + SEARCH_TOKEN_COLLECTIONS.map(async ({ name, fields }) => { + const collection = connection.db!.collection(name); + const indexes = await collection.indexes().catch((error: { codeName?: string }) => { + /** No collection yet: nothing to search, nothing to migrate. */ + if (error?.codeName === 'NamespaceNotFound') { + return null; + } + throw error; + }); + if (indexes == null) { + return null; + } + const built = new Set(indexes.map((index) => JSON.stringify(index.key))); + const unindexed = fields.some((field) => + searchTokenIndexes(field).some((index) => !built.has(JSON.stringify(index))), + ); + /** Without the indexes the probe below would scan; the warning is already due. */ + if (unindexed) { + return name; + } + const unmigrated = await collection.findOne(missingTokens(fields), { + projection: { _id: 1 }, + maxTimeMS: STARTUP_PROBE_MAX_TIME_MS, + }); + return unmigrated != null ? name : null; + }), + ); + const names = pending.filter((name): name is string => name != null); + if (names.length > 0) { + logger.warn( + `[SearchTokenMigration] Some ${names.join(' and ')} lack search tokens or their indexes; people search scans the collection until you run: npm run migrate:search-tokens`, + ); + } + } catch (error) { + logger.error('[SearchTokenMigration] Failed to check search token migration:', error); + } +} diff --git a/packages/data-schemas/src/models/plugins/searchTokens.spec.ts b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts new file mode 100644 index 0000000000..edd1c6154d --- /dev/null +++ b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts @@ -0,0 +1,318 @@ +import mongoose from 'mongoose'; +import { PrincipalType } from 'librechat-data-provider'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type * as t from '~/types'; +import { createUserGroupMethods } from '~/methods/userGroup'; +import { tenantStorage } from '~/config/tenantContext'; +import { buildUserSearchFilter } from '~/utils/search'; +import { createModels } from '~/models'; + +jest.mock('~/config/winston', () => ({ + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), +})); + +const TOKENS = '+nameTokens +emailTokens +usernameTokens'; + +let mongoServer: MongoMemoryServer; +let User: mongoose.Model; +let Group: mongoose.Model; +let methods: ReturnType; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + const models = createModels(mongoose); + User = models.User; + Group = models.Group as mongoose.Model; + methods = createUserGroupMethods(mongoose); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(async () => { + await mongoose.connection.dropDatabase(); +}); + +const user = (name: string, email: string, username = '', extra: Partial = {}) => ({ + name, + email, + username, + provider: 'local', + ...extra, +}); + +async function tokensOf(id: unknown) { + return User.findById(id).select(TOKENS).lean(); +} + +async function userNames(query: string, limit = 10): Promise { + const results = await methods.searchPrincipals(query, limit, [PrincipalType.USER]); + return results.map((r) => r.name ?? '').sort(); +} + +describe('search token write paths', () => { + it('derives tokens on create and on save of a changed name', async () => { + const doc = await User.create(user('José Álvarez', 'Jose.Alvarez@Example.org', 'jalvarez')); + expect(await tokensOf(doc._id)).toMatchObject({ + nameTokens: ['jose', 'alvarez'], + emailTokens: ['jose.alvarez@example.org', 'jose', 'alvarez', 'example', 'org'], + usernameTokens: ['jalvarez'], + }); + + doc.name = 'Joe Bloggs'; + await doc.save(); + const saved = await tokensOf(doc._id); + expect(saved?.nameTokens).toEqual(['joe', 'bloggs']); + expect(saved?.usernameTokens).toEqual(['jalvarez']); + }); + + it('updates tokens through findOneAndUpdate, findByIdAndUpdate, updateOne and updateMany', async () => { + const [a, b] = await User.create([user('Ann A', 'a@x.io'), user('Ben B', 'b@x.io')]); + + await User.findOneAndUpdate({ _id: a._id }, { $set: { name: 'Carla Diaz' } }); + expect((await tokensOf(a._id))?.nameTokens).toEqual(['carla', 'diaz']); + + await User.findByIdAndUpdate(a._id, { email: 'carla@new.dev' }); + expect((await tokensOf(a._id))?.emailTokens).toEqual(['carla@new.dev', 'carla', 'new', 'dev']); + + await User.updateOne({ _id: b._id }, { $set: { username: 'ben_b' } }); + expect((await tokensOf(b._id))?.usernameTokens).toEqual(['ben_b', 'ben', 'b']); + + await User.updateMany({}, { $unset: { name: '' } }); + expect((await tokensOf(a._id))?.nameTokens).toEqual([]); + expect((await tokensOf(b._id))?.nameTokens).toEqual([]); + }); + + it('sets tokens on upsert inserts and insertMany', async () => { + await User.updateOne( + { email: 'new@x.io' }, + { $setOnInsert: { name: 'New Person', email: 'new@x.io', provider: 'local' } }, + { upsert: true }, + ); + const upserted = await User.findOne({ email: 'new@x.io' }).select(TOKENS).lean(); + expect(upserted?.nameTokens).toEqual(['new', 'person']); + expect(upserted?.usernameTokens).toEqual([]); + + await User.updateOne( + { email: 'filter@x.io' }, + { $set: { name: 'From Filter', provider: 'local' } }, + { upsert: true }, + ); + const fromFilter = await User.findOne({ email: 'filter@x.io' }).select(TOKENS).lean(); + expect(fromFilter?.emailTokens).toEqual(['filter@x.io', 'filter', 'x', 'io']); + + await User.insertMany([user('Many One', 'one@x.io', 'm1')]); + const inserted = await User.findOne({ email: 'one@x.io' }).select(TOKENS).lean(); + expect(inserted?.nameTokens).toEqual(['many', 'one']); + expect(inserted?.usernameTokens).toEqual(['m1']); + }); + + it('keeps tokens out of default reads', async () => { + const doc = await User.create(user('Hidden Tokens', 'h@x.io')); + const plain = await User.findById(doc._id).lean(); + expect(plain).not.toHaveProperty('nameTokens'); + }); + + it('updates group tokens through create, upsertGroupByExternalId and updateGroupById', async () => { + const group = await methods.createGroup({ name: 'Core Platform', source: 'local' }); + const read = () => + Group.findById(group._id).select('+nameTokens +emailTokens').lean(); + expect((await read())?.nameTokens).toEqual(['core', 'platform']); + + await methods.updateGroupById(group._id, { name: 'Data Science', email: 'ds@corp.io' }); + expect(await read()).toMatchObject({ + nameTokens: ['data', 'science'], + emailTokens: ['ds@corp.io', 'ds', 'corp', 'io'], + }); + + const entra = await methods.upsertGroupByExternalId('ext-1', 'entra', { name: 'Sales EMEA' }); + const entraTokens = await Group.findById(entra?._id).select('+nameTokens').lean(); + expect(entraTokens?.nameTokens).toEqual(['sales', 'emea']); + expect(await Group.countDocuments({ _id: entra?._id, emailTokens: { $exists: true } })).toBe(1); + }); + + it('keeps tokens when an update assigns undefined to a source field', async () => { + const doc = await User.create(user('Kept Name', 'kept@x.io')); + await User.findByIdAndUpdate(doc._id, { $set: { name: undefined, role: 'ADMIN' } }); + const stored = await User.findById(doc._id).select(`name ${TOKENS}`).lean(); + expect(stored?.name).toBe('Kept Name'); + expect(stored?.nameTokens).toEqual(['kept', 'name']); + }); +}); + +describe('searchPrincipals word-prefix search', () => { + beforeEach(async () => { + await User.create([ + user('John Smith', 'john.smith@acme.com', 'jsmith'), + user('Johanna Müller', 'jm@globex.io', 'hanna_m'), + user('Bob Johnson', 'bob@initech.net', 'bobby'), + user('Zoë Ng', 'zoe@acme.com', 'zng'), + ]); + }); + + it('matches the prefix of a first name, a last name, an email word, a domain and a username', async () => { + expect(await userNames('joh')).toEqual(['Bob Johnson', 'Johanna Müller', 'John Smith']); + expect(await userNames('smi')).toEqual(['John Smith']); + expect(await userNames('mull')).toEqual(['Johanna Müller']); + expect(await userNames('acme')).toEqual(['John Smith', 'Zoë Ng']); + expect(await userNames('hanna')).toEqual(['Johanna Müller']); + expect(await userNames('john.smith@ac')).toEqual(['John Smith']); + }); + + it('requires every query token, across fields', async () => { + expect(await userNames('john smi')).toEqual(['John Smith']); + expect(await userNames('smith john')).toEqual(['John Smith']); + expect(await userNames('jo acme')).toEqual(['John Smith']); + expect(await userNames('john globex')).toEqual([]); + }); + + it('is case- and accent-insensitive', async () => { + expect(await userNames('ZOE')).toEqual(['Zoë Ng']); + expect(await userNames('MÜLLER')).toEqual(['Johanna Müller']); + }); + + it('does not match inside a word (intended: only word prefixes are index-bound)', async () => { + expect(await userNames('ohn')).toEqual([]); + expect(await userNames('mith')).toEqual([]); + }); + + it('respects the limit', async () => { + expect(await userNames('jo', 2)).toHaveLength(2); + }); + + it('finds documents written before tokens existed through the substring fallback', async () => { + await User.collection.insertOne({ name: 'Legacy Person', email: 'legacy@old.io' }); + expect(await userNames('gacy')).toEqual(['Legacy Person']); + }); + + it('stays inside the active tenant', async () => { + await tenantStorage.run({ tenantId: 'tenant-a' }, () => + User.create(user('John Tenant', 'john@tenant-a.io')), + ); + await tenantStorage.run({ tenantId: 'tenant-b' }, () => + User.create(user('John Other', 'john@tenant-b.io')), + ); + const names = await tenantStorage.run({ tenantId: 'tenant-a' }, () => userNames('john')); + expect(names).toEqual(['John Tenant']); + }); + + it('finds groups by name and email word prefix', async () => { + await Group.create([ + { name: 'Platform Team', source: 'local', email: 'platform@corp.io' }, + { name: 'Design', source: 'local', description: 'Platform design' }, + ]); + const groups = await methods.searchPrincipals('plat', 10, [PrincipalType.GROUP]); + expect(groups.map((g) => g.name)).toEqual(['Platform Team']); + }); +}); + +describe('search token serialization', () => { + it('keeps token fields out of the JSON and object output of a freshly created document', async () => { + const [group] = await Group.create([{ name: 'Platform Team', source: 'local' }]); + const created = await User.create(user('Ana Lima', 'ana@x.io', 'ana')); + for (const doc of [group, created]) { + expect(doc.get('nameTokens')).toBeDefined(); + expect(doc.toJSON()).not.toHaveProperty('nameTokens'); + expect(doc.toObject()).not.toHaveProperty('nameTokens'); + expect(doc.toObject()).not.toHaveProperty('emailTokens'); + } + const stored = await Group.findById(group._id).select('+nameTokens').lean(); + expect(stored?.nameTokens).toEqual(['platform', 'team']); + }); +}); + +describe('search query plan', () => { + const planStages = (plan: unknown): string[] => { + const stages: string[] = []; + const visit = (node: unknown) => { + if (node == null || typeof node !== 'object') { + return; + } + const record = node as Record; + if (typeof record.stage === 'string') { + stages.push(record.stage); + } + for (const value of Object.values(record)) { + visit(value); + } + }; + visit(plan); + return stages; + }; + + beforeEach(async () => { + await User.createIndexes(); + await User.create([ + user('John Smith', 'john.smith@acme.com', 'jsmith'), + user('Bob Johnson', 'bob@initech.net', 'bobby'), + ]); + }); + + it.each(['john', 'john smi', 'john.smith@acme'])( + 'uses index scans, never a collection scan, for "%s"', + async (query) => { + const filter = buildUserSearchFilter(query)!; + const explain = (await User.find(filter).limit(10).explain('queryPlanner')) as unknown as { + queryPlanner: { winningPlan: unknown }; + }; + const stages = planStages(explain.queryPlanner.winningPlan); + expect(stages).toContain('IXSCAN'); + expect(stages).not.toContain('COLLSCAN'); + }, + ); + + it('uses index scans with the admin search sort by name', async () => { + const filter = buildUserSearchFilter('jo')!; + const explain = (await User.find(filter) + .sort({ name: 1 }) + .limit(20) + .explain('queryPlanner')) as unknown as { queryPlanner: { winningPlan: unknown } }; + const stages = planStages(explain.queryPlanner.winningPlan); + expect(stages).toContain('IXSCAN'); + expect(stages).not.toContain('COLLSCAN'); + }); + + it('uses index scans under a tenant scope', async () => { + const filter = buildUserSearchFilter('john')!; + const explain = (await tenantStorage.run({ tenantId: 'tenant-a' }, () => + User.find(filter).limit(10).explain('queryPlanner').exec(), + )) as unknown as { queryPlanner: { winningPlan: unknown } }; + const stages = planStages(explain.queryPlanner.winningPlan); + expect(stages).toContain('IXSCAN'); + expect(stages).not.toContain('COLLSCAN'); + }); + + it('seeks only the scoped tenant keys when every tenant shares the prefix', async () => { + /** The scoped tenant is large with two matches; every other tenant is all matches, + * so neither a tenant-only nor a token-first seek can stay small. */ + const doc = (name: string, i: number, tenant: string) => ({ + name: `${name} ${i}`, + email: `${name.toLowerCase()}${i}@${tenant}.io`, + tenantId: tenant, + nameTokens: [name.toLowerCase(), String(i)], + emailTokens: [ + `${name.toLowerCase()}${i}@${tenant}.io`, + `${name.toLowerCase()}${i}`, + tenant, + 'io', + ], + usernameTokens: [], + }); + await User.collection.insertMany([ + ...Array.from({ length: 300 }, (_, i) => doc(i < 2 ? 'Johnny' : 'Anna', i, 'tenant-3')), + ...Array.from({ length: 300 }, (_, i) => doc('Johnny', i, `tenant-${10 + (i % 10)}`)), + ]); + const filter = buildUserSearchFilter('johnny')!; + const explain = (await tenantStorage.run({ tenantId: 'tenant-3' }, () => + User.find(filter).explain('executionStats').exec(), + )) as unknown as { executionStats: { nReturned: number; totalKeysExamined: number } }; + expect(explain.executionStats.nReturned).toBe(2); + expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(10); + }); +}); diff --git a/packages/data-schemas/src/models/plugins/searchTokens.ts b/packages/data-schemas/src/models/plugins/searchTokens.ts new file mode 100644 index 0000000000..67b935b57b --- /dev/null +++ b/packages/data-schemas/src/models/plugins/searchTokens.ts @@ -0,0 +1,71 @@ +import type { Schema, Query } from 'mongoose'; +import type { SearchTokenField } from '~/utils/search'; +import { withSearchTokens, searchTokenIndexes, computeSearchTokens } from '~/utils/search'; + +/** + * Adds the derived search-token arrays of `fields` to `schema`, indexes each + * with `tenantId` in both key orders (see `searchTokenIndexes`), and keeps them in sync on `save`, `insertMany` and the + * update queries. `bulkWrite` runs no middleware: callers that change a source + * field through it must apply `withSearchTokens` themselves. + * + * Token fields are `select: false` and stripped from `toJSON`/`toObject` + * output (a freshly saved document still holds them); they exist for the + * index, not for readers. + */ +export function applySearchTokens(schema: Schema, fields: readonly SearchTokenField[]): void { + for (const field of fields) { + schema.add({ [field.tokens]: { type: [String], default: undefined, select: false } }); + for (const index of searchTokenIndexes(field)) { + schema.index(index); + } + } + + for (const option of ['toJSON', 'toObject'] as const) { + schema.set(option, { + ...schema.get(option), + transform(_doc, ret: Record) { + for (const field of fields) { + delete ret[field.tokens]; + } + return ret; + }, + }); + } + + schema.pre('save', function () { + for (const field of fields) { + if (this.isNew || this.isModified(field.source)) { + this.set(field.tokens, computeSearchTokens(field.kind, this.get(field.source))); + } + } + }); + + schema.pre( + ['findOneAndUpdate', 'updateOne', 'updateMany'], + { document: false, query: true }, + function (this: Query) { + const update = this.getUpdate(); + if (!update) { + return; + } + const next = withSearchTokens(fields, update, { + upsert: this.getOptions().upsert === true, + filter: this.getFilter(), + }); + if (next !== update) { + this.setUpdate(next); + } + }, + ); + + schema.pre('insertMany', function (next, docs: unknown) { + if (Array.isArray(docs)) { + for (const doc of docs as Array>) { + for (const field of fields) { + doc[field.tokens] = computeSearchTokens(field.kind, doc[field.source]); + } + } + } + next(); + }); +} diff --git a/packages/data-schemas/src/schema/group.ts b/packages/data-schemas/src/schema/group.ts index bb19e4aa81..17c5e1bd06 100644 --- a/packages/data-schemas/src/schema/group.ts +++ b/packages/data-schemas/src/schema/group.ts @@ -1,5 +1,7 @@ import { Schema } from 'mongoose'; import type { IGroup } from '~/types'; +import { applySearchTokens } from '~/models/plugins/searchTokens'; +import { GROUP_SEARCH_TOKEN_FIELDS } from '~/utils/search'; const groupSchema: Schema = new Schema( { @@ -57,5 +59,6 @@ groupSchema.index( }, ); groupSchema.index({ memberIds: 1, tenantId: 1 }); +applySearchTokens(groupSchema, GROUP_SEARCH_TOKEN_FIELDS); export default groupSchema; diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.ts index 87ba658ce0..0b7329d322 100644 --- a/packages/data-schemas/src/schema/user.ts +++ b/packages/data-schemas/src/schema/user.ts @@ -1,5 +1,7 @@ import { Schema } from 'mongoose'; import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider'; +import { applySearchTokens } from '~/models/plugins/searchTokens'; +import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search'; import { IUser } from '~/types'; // Session sub-schema @@ -244,6 +246,7 @@ userSchema.index({ idOnTheSource: 1, openidIssuer: 1, tenantId: 1 }); them. The rare cleanup that pulls a deleted agent from every user names the tenant too; the one that does not (`$in` over a batch of ids) is a maintenance write, not a page. */ userSchema.index({ tenantId: 1, 'favorites.agentId': 1 }); +applySearchTokens(userSchema, USER_SEARCH_TOKEN_FIELDS); const oAuthIdFields = [ 'googleId', diff --git a/packages/data-schemas/src/types/group.ts b/packages/data-schemas/src/types/group.ts index 15cb6f288e..f935b9837f 100644 --- a/packages/data-schemas/src/types/group.ts +++ b/packages/data-schemas/src/types/group.ts @@ -15,6 +15,9 @@ export interface IGroup extends Document { createdAt?: Date; updatedAt?: Date; tenantId?: string; + /** Derived search tokens (see `~/utils/search`); `select: false`. */ + nameTokens?: string[]; + emailTokens?: string[]; } export interface CreateGroupRequest { diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.ts index e2b3cd861b..9b0e8837b8 100644 --- a/packages/data-schemas/src/types/user.ts +++ b/packages/data-schemas/src/types/user.ts @@ -89,6 +89,10 @@ export interface IUser extends Document { /** Field for external source identification (for consistency with TPrincipal schema) */ idOnTheSource?: string; tenantId?: string; + /** Derived search tokens (see `~/utils/search`); `select: false`. */ + nameTokens?: string[]; + emailTokens?: string[]; + usernameTokens?: string[]; federatedTokens?: OIDCTokens; openidTokens?: OIDCTokens; } diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts index f5a4c6fd48..828eb80429 100644 --- a/packages/data-schemas/src/utils/index.ts +++ b/packages/data-schemas/src/utils/index.ts @@ -1,5 +1,6 @@ export * from './principal'; export * from './string'; +export * from './search'; export * from './tempChatRetention'; export * from './retention'; export { tenantSafeBulkWrite } from './tenantBulkWrite'; diff --git a/packages/data-schemas/src/utils/search.spec.ts b/packages/data-schemas/src/utils/search.spec.ts new file mode 100644 index 0000000000..db38aebcb6 --- /dev/null +++ b/packages/data-schemas/src/utils/search.spec.ts @@ -0,0 +1,215 @@ +import { + withSearchTokens, + computeSearchTokens, + normalizeSearchText, + buildUserSearchFilter, + MAX_SEARCH_TOKENS, + USER_SEARCH_TOKEN_FIELDS, + MAX_SEARCH_TOKEN_LENGTH, + MAX_SEARCH_QUERY_TOKENS, +} from './search'; + +describe('normalizeSearchText', () => { + it('lowercases and folds accents and compatibility forms', () => { + expect(normalizeSearchText('José ÅNGSTRÖM')).toBe('jose angstrom'); + expect(normalizeSearchText('Full')).toBe('full'); + expect(normalizeSearchText('İstanbul')).toBe('istanbul'); + }); +}); + +describe('computeSearchTokens', () => { + it('splits names on whitespace and punctuation', () => { + expect(computeSearchTokens('words', " Mary-Jane O'Brien, Jr. ")).toEqual([ + 'mary', + 'jane', + 'o', + 'brien', + 'jr', + ]); + }); + + it('keeps non-Latin scripts as words', () => { + expect(computeSearchTokens('words', '山田 太郎')).toEqual(['山田', '太郎']); + expect(computeSearchTokens('words', 'Ζωή Κ.')).toEqual(['ζωη', 'κ']); + }); + + it('indexes the full email, local-part words, and domain labels', () => { + expect(computeSearchTokens('email', 'John.Smith+work@Mail.Example-Co.com')).toEqual([ + 'john.smith+work@mail.example-co.com', + 'john', + 'smith', + 'work', + 'mail', + 'example', + 'co', + 'com', + 'example-co', + ]); + }); + + it('indexes the full username and its parts', () => { + expect(computeSearchTokens('handle', 'jane_doe.99')).toEqual([ + 'jane_doe.99', + 'jane', + 'doe', + '99', + ]); + }); + + it('dedupes and drops empties', () => { + expect(computeSearchTokens('words', 'Ann ann ANN --')).toEqual(['ann']); + expect(computeSearchTokens('words', ' ')).toEqual([]); + expect(computeSearchTokens('words', '')).toEqual([]); + }); + + it('returns no tokens for non-strings', () => { + expect(computeSearchTokens('words', undefined)).toEqual([]); + expect(computeSearchTokens('email', null)).toEqual([]); + expect(computeSearchTokens('handle', 42)).toEqual([]); + }); + + it('caps token length and token count', () => { + const long = 'a'.repeat(MAX_SEARCH_TOKEN_LENGTH + 10); + expect(computeSearchTokens('words', long)).toEqual(['a'.repeat(MAX_SEARCH_TOKEN_LENGTH)]); + + const many = Array.from({ length: MAX_SEARCH_TOKENS + 10 }, (_, i) => `w${i}`).join(' '); + const tokens = computeSearchTokens('words', many); + expect(tokens).toHaveLength(MAX_SEARCH_TOKENS); + expect(tokens[0]).toBe('w0'); + }); + + it('does not split a surrogate pair when truncating', () => { + const [token] = computeSearchTokens('words', '\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH + 5)); + expect(Array.from(token)).toHaveLength(MAX_SEARCH_TOKEN_LENGTH); + expect(token).toBe('\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH)); + }); +}); + +describe('withSearchTokens', () => { + it('derives tokens for each source field in $set and leaves the input untouched', () => { + const update = { $set: { name: 'Ana Lima', role: 'USER' } }; + const next = withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update); + expect(next).toEqual({ $set: { name: 'Ana Lima', role: 'USER', nameTokens: ['ana', 'lima'] } }); + expect(update).toEqual({ $set: { name: 'Ana Lima', role: 'USER' } }); + }); + + it('handles top-level, $setOnInsert and $unset writes', () => { + expect( + withSearchTokens(USER_SEARCH_TOKEN_FIELDS, { + username: 'al', + $setOnInsert: { email: 'a@b.io' }, + $unset: { name: '' }, + }), + ).toEqual({ + username: 'al', + usernameTokens: ['al'], + $setOnInsert: { email: 'a@b.io', emailTokens: ['a@b.io', 'a', 'b', 'io'] }, + $unset: { name: '' }, + $set: { nameTokens: [] }, + }); + }); + + it('ignores undefined assignments, which Mongoose drops from the update', () => { + const update = { $set: { name: undefined, role: 'ADMIN' }, username: undefined }; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update); + }); + + it('ignores an undefined $unset, which Mongoose also drops', () => { + const update = { $unset: { name: undefined } }; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update); + }); + + it('initializes every unwritten token field on upsert inserts', () => { + expect( + withSearchTokens(USER_SEARCH_TOKEN_FIELDS, { $set: { name: 'Al' } }, { upsert: true }), + ).toEqual({ + $set: { name: 'Al', nameTokens: ['al'] }, + $setOnInsert: { emailTokens: [], usernameTokens: [] }, + }); + }); + + it('takes upsert insert tokens from equality conditions the insert copies', () => { + expect( + withSearchTokens( + USER_SEARCH_TOKEN_FIELDS, + { $set: { name: 'Al' } }, + { upsert: true, filter: { email: 'al@x.io', username: { $eq: 'al' } } }, + ), + ).toEqual({ + $set: { name: 'Al', nameTokens: ['al'] }, + $setOnInsert: { emailTokens: ['al@x.io', 'al', 'x', 'io'], usernameTokens: ['al'] }, + }); + expect( + withSearchTokens( + USER_SEARCH_TOKEN_FIELDS, + { $set: { name: 'Al' } }, + { upsert: true, filter: { email: { $in: ['a@x.io', 'b@x.io'] } } }, + ), + ).toEqual({ + $set: { name: 'Al', nameTokens: ['al'] }, + $setOnInsert: { usernameTokens: [] }, + }); + }); + + it('returns the same reference when no source field is written', () => { + const update = { $set: { role: 'ADMIN' } }; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update); + const pipeline = [{ $set: { name: 'x' } }]; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, pipeline)).toBe(pipeline); + }); +}); + +describe('buildUserSearchFilter', () => { + it('returns null when the query has nothing to match', () => { + expect(buildUserSearchFilter('')).toBeNull(); + expect(buildUserSearchFilter(' ')).toBeNull(); + }); + + it('matches a single token by anchored, case-sensitive prefix on every token field', () => { + const filter = buildUserSearchFilter(' José ') as { $or: Array> }; + const prefixes = filter.$or.filter((branch) => Object.keys(branch).length === 1); + expect(prefixes).toEqual([ + { nameTokens: /^jose/ }, + { emailTokens: /^jose/ }, + { usernameTokens: /^jose/ }, + ]); + expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /José/i }); + }); + + it('anchors the legacy fallback for callers whose old search was a prefix match', () => { + const filter = buildUserSearchFilter('mith', { legacyPrefix: true })!; + expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /^mith/i }); + expect(filter.$or).not.toContainEqual({ nameTokens: { $exists: false }, name: /mith/i }); + }); + + it('requires every query token and also tries the whole query on full-value fields', () => { + const filter = buildUserSearchFilter('john smi') as { $and: Array<{ $or: unknown[] }> }; + expect(filter.$and).toHaveLength(2); + expect(filter.$and[0].$or).toContainEqual({ nameTokens: /^john/ }); + expect(filter.$and[1].$or).toContainEqual({ nameTokens: /^smi/ }); + expect(filter.$and[1].$or).toContainEqual({ emailTokens: /^john smi/ }); + expect(filter.$and[1].$or).not.toContainEqual({ nameTokens: /^john smi/ }); + }); + + it('escapes regex metacharacters and requires every query token up to the cap', () => { + const filter = buildUserSearchFilter('a.b') as { $and: Array<{ $or: unknown[] }> }; + expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^a\.b/ }); + + const words = (n: number) => Array.from({ length: n }, (_, i) => `t${i}`).join(' '); + const all = buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS)) as { + $and: Array<{ $or: unknown[] }>; + }; + expect(all.$and).toHaveLength(MAX_SEARCH_QUERY_TOKENS); + expect(all.$and[MAX_SEARCH_QUERY_TOKENS - 1].$or).toContainEqual({ + nameTokens: new RegExp(`^t${MAX_SEARCH_QUERY_TOKENS - 1}`), + }); + /** Over the cap, nothing matches: extra words are never silently dropped. */ + expect(buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS + 1))).toBeNull(); + }); + + it('searches punctuation-only queries on full values only', () => { + const filter = buildUserSearchFilter('.*') as { $or: unknown[] }; + expect(filter.$or).toContainEqual({ emailTokens: /^\.\*/ }); + expect(filter.$or).not.toContainEqual({ nameTokens: expect.anything() }); + }); +}); diff --git a/packages/data-schemas/src/utils/search.ts b/packages/data-schemas/src/utils/search.ts new file mode 100644 index 0000000000..63a232561d --- /dev/null +++ b/packages/data-schemas/src/utils/search.ts @@ -0,0 +1,328 @@ +import { escapeRegExp } from './string'; + +/** + * Word-prefix search over derived token arrays. + * + * Each searchable source field (`name`, `email`, `username`) gets a lowercase, + * accent-folded token array stored beside it and indexed with `tenantId`. A query + * is tokenized the same way, and each query token must prefix-match some token, + * so "john smi" finds "John Smith" through an anchored, case-sensitive regex + * that the multikey index can bound. Matches inside a word ("ohn") are not + * found, by design: that is the price of an index-bound search. + * + * The token rules are a stored format: changing them requires re-running the + * backfill, so the caps are constants rather than configuration. + */ + +/** How a source value is split into tokens. */ +export type SearchTokenKind = 'words' | 'email' | 'handle'; + +export interface SearchTokenField { + /** The stored field the tokens derive from. */ + readonly source: string; + /** The derived, indexed token array. */ + readonly tokens: string; + readonly kind: SearchTokenKind; +} + +export const USER_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [ + { source: 'name', tokens: 'nameTokens', kind: 'words' }, + { source: 'email', tokens: 'emailTokens', kind: 'email' }, + { source: 'username', tokens: 'usernameTokens', kind: 'handle' }, +]; + +export const GROUP_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [ + { source: 'name', tokens: 'nameTokens', kind: 'words' }, + { source: 'email', tokens: 'emailTokens', kind: 'email' }, +]; + +/** Longest stored or queried word token; longer words are truncated to this prefix. */ +export const MAX_SEARCH_TOKEN_LENGTH = 64; +/** Longest whole-value token (a full email address or username). */ +export const MAX_SEARCH_VALUE_LENGTH = 256; +/** Most tokens stored per field. */ +export const MAX_SEARCH_TOKENS = 32; +/** + * Most distinct query words. Every word must match, so a longer query matches + * nothing rather than ignoring its extra words; the cap bounds the filter a + * single request can build. + */ +export const MAX_SEARCH_QUERY_TOKENS = 16; + +const NON_WORD = /[^\p{L}\p{N}]+/u; +const COMBINING_MARKS = /\p{M}+/gu; + +function truncate(value: string, max: number): string { + if (value.length <= max) { + return value; + } + return Array.from(value).slice(0, max).join(''); +} + +/** Lowercases and folds accents, so "José" and "jose" compare equal. */ +export function normalizeSearchText(value: string): string { + return value.normalize('NFKD').replace(COMBINING_MARKS, '').toLowerCase(); +} + +function words(normalized: string): string[] { + return normalized + .split(NON_WORD) + .filter(Boolean) + .map((word) => truncate(word, MAX_SEARCH_TOKEN_LENGTH)); +} + +/** + * A value reduced to the words the filter matches, joined by single spaces, so + * relevance scoring sees "Mary-Jane" and "mary jane" as the same text. + */ +export function normalizeSearchPhrase(value: string): string { + return words(normalizeSearchText(value)).join(' '); +} + +/** + * Relevance of a matched document for `query`, over the values the filter + * searched. Like the filter, it ignores word order: 100 when a value has the + * query's words in any order, 80 when a value starts with the query, 50 when a + * value contains it or every query word prefixes some word (in one value or + * across them), 10 otherwise. + */ +export function scoreSearchMatch( + values: ReadonlyArray, + query: string, +): number { + const phrase = normalizeSearchPhrase(query); + if (!phrase) { + return 10; + } + const queryWords = phrase.split(' '); + const sortedQuery = [...queryWords].sort().join(' '); + const valueWords: string[] = []; + let score = 10; + for (const value of values) { + if (!value) { + continue; + } + const field = normalizeSearchPhrase(value); + const words = field.split(' '); + if (field === phrase || [...words].sort().join(' ') === sortedQuery) { + return 100; + } + if (field.startsWith(phrase)) { + score = Math.max(score, 80); + } else if (field.includes(phrase)) { + score = Math.max(score, 50); + } + valueWords.push(...words); + } + const everyWord = queryWords.every((queryWord) => + valueWords.some((word) => word.startsWith(queryWord)), + ); + return everyWord ? Math.max(score, 50) : score; +} + +/** Derives the stored token array for one field value. Non-strings yield no tokens. */ +export function computeSearchTokens(kind: SearchTokenKind, value: unknown): string[] { + if (typeof value !== 'string') { + return []; + } + const normalized = normalizeSearchText(value.trim()); + if (!normalized) { + return []; + } + const tokens: string[] = []; + if (kind !== 'words') { + tokens.push(truncate(normalized, MAX_SEARCH_VALUE_LENGTH)); + } + tokens.push(...words(normalized)); + if (kind === 'email') { + const domain = normalized.slice(normalized.lastIndexOf('@') + 1); + tokens.push(...domain.split('.').filter(Boolean)); + } + return [...new Set(tokens)].slice(0, MAX_SEARCH_TOKENS); +} + +/** Every token field of `fields`, computed from a document's source values. */ +export function computeSearchTokenSet( + fields: readonly SearchTokenField[], + doc: Record, +): Record { + const result: Record = {}; + for (const field of fields) { + result[field.tokens] = computeSearchTokens(field.kind, doc[field.source]); + } + return result; +} + +type UpdateDoc = Record; + +/** + * The indexes behind each token field. Tenant first serves tenant-scoped + * searches with an equality seek before the prefix range; token first serves + * unscoped ones (single-tenant deployments run without a tenant context), which + * could not use the tenant-first index at all. + */ +export function searchTokenIndexes(field: SearchTokenField): Array> { + return [ + { tenantId: 1, [field.tokens]: 1 }, + { [field.tokens]: 1, tenantId: 1 }, + ]; +} + +/** Mongoose drops `undefined` assignments from an update, so they write nothing. */ +function writes(doc: UpdateDoc | undefined, key: string): doc is UpdateDoc { + return doc != null && key in doc && doc[key] !== undefined; +} + +const SKIP = Symbol('skip'); + +/** + * The value an upsert insert takes for `key` from the query filter: a plain or + * `$eq` equality is copied into the new document, no condition leaves it unset. + * Any other condition is ambiguous, so the token field is left for the backfill. + */ +function insertedValue(filter: UpdateDoc | undefined, key: string): unknown { + const condition = filter?.[key]; + if (condition === undefined) { + return undefined; + } + if (typeof condition === 'string') { + return condition; + } + const eq = operator({ condition }, 'condition'); + return eq && Object.keys(eq).length === 1 && typeof eq.$eq === 'string' ? eq.$eq : SKIP; +} + +function operator(update: UpdateDoc, key: string): UpdateDoc | undefined { + const value = update[key]; + return value != null && typeof value === 'object' && !Array.isArray(value) + ? (value as UpdateDoc) + : undefined; +} + +/** + * Returns `update` with the token fields recomputed for every source field it + * writes, through `$set`, `$setOnInsert`, `$unset` or a top-level replacement + * value. Each token field depends on one source field, so a partial update + * stays correct. On an upsert, a token field whose source the update does not + * write is initialized on insert from the source's equality condition in + * `filter` (MongoDB copies it into the new document), or to `[]` when the + * filter has none, so new documents never look un-migrated. Copy-on-write: the same reference comes back when nothing + * changes. Pipeline updates are returned untouched. + * + * Mongoose runs no middleware for `bulkWrite`; a bulk write that changes a + * source field must pass its update through this function. + */ +export function withSearchTokens( + fields: readonly SearchTokenField[], + update: T, + options: { upsert?: boolean; filter?: UpdateDoc } = {}, +): T { + if (update == null || typeof update !== 'object' || Array.isArray(update)) { + return update; + } + const source = update as UpdateDoc; + let next: UpdateDoc | undefined; + const write = (op: string | null, key: string, tokens: string[]) => { + next ??= { ...source }; + if (op == null) { + next[key] = tokens; + return; + } + next[op] = { ...(operator(next, op) ?? {}), [key]: tokens }; + }; + + for (const field of fields) { + const set = operator(source, '$set'); + const setOnInsert = operator(source, '$setOnInsert'); + const unset = operator(source, '$unset'); + let written = true; + if (writes(set, field.source)) { + write('$set', field.tokens, computeSearchTokens(field.kind, set[field.source])); + } else if (writes(source, field.source)) { + write(null, field.tokens, computeSearchTokens(field.kind, source[field.source])); + } else if (writes(unset, field.source)) { + write('$set', field.tokens, []); + } else { + written = false; + } + if (writes(setOnInsert, field.source)) { + write( + '$setOnInsert', + field.tokens, + computeSearchTokens(field.kind, setOnInsert[field.source]), + ); + } else if (options.upsert && !written) { + const condition = insertedValue(options.filter, field.source); + if (condition !== SKIP) { + write('$setOnInsert', field.tokens, computeSearchTokens(field.kind, condition)); + } + } + } + return (next ?? source) as T; +} + +export interface SearchFilterOptions { + /** Anchor the legacy fallback at the start of the field, for callers whose old search was a prefix match. */ + legacyPrefix?: boolean; +} + +/** + * Builds the filter for a word-prefix search, or `null` when the query has + * nothing to match. + * + * Every query token must prefix-match a token of some field (fields may + * differ per token, so "john gmail" finds John at gmail.com). Fields that + * store their whole value (email, username) also match the whole query as a + * prefix, so a pasted "john.smith@exa" works. Documents written before the + * token fields existed fall back to the caller's old case-insensitive regex + * on the source field (unanchored, or anchored with `legacyPrefix`); the + * fallback branches seek the token index on `$exists: false` and match + * nothing once the backfill has run. + */ +export function buildSearchTokenFilter( + fields: readonly SearchTokenField[], + query: string, + options: SearchFilterOptions = {}, +): Record | null { + const trimmed = query.trim(); + const normalized = normalizeSearchText(trimmed); + const queryTokens = [...new Set(words(normalized))]; + const whole = truncate(normalized, MAX_SEARCH_VALUE_LENGTH); + const matchWhole = whole.length > 0 && !(queryTokens.length === 1 && queryTokens[0] === whole); + if ( + !trimmed || + queryTokens.length > MAX_SEARCH_QUERY_TOKENS || + (queryTokens.length === 0 && !matchWhole) + ) { + return null; + } + + const legacy = new RegExp(`${options.legacyPrefix ? '^' : ''}${escapeRegExp(trimmed)}`, 'i'); + const shared: Record[] = []; + for (const field of fields) { + if (matchWhole && field.kind !== 'words') { + shared.push({ [field.tokens]: new RegExp(`^${escapeRegExp(whole)}`) }); + } + shared.push({ [field.tokens]: { $exists: false }, [field.source]: legacy }); + } + + if (queryTokens.length === 0) { + return { $or: shared }; + } + const clauses = queryTokens.map((token) => { + const prefix = new RegExp(`^${escapeRegExp(token)}`); + return { $or: [...fields.map((field) => ({ [field.tokens]: prefix })), ...shared] }; + }); + return clauses.length === 1 ? clauses[0] : { $and: clauses }; +} + +export function buildUserSearchFilter( + query: string, + options?: SearchFilterOptions, +): Record | null { + return buildSearchTokenFilter(USER_SEARCH_TOKEN_FIELDS, query, options); +} + +export function buildGroupSearchFilter(query: string): Record | null { + return buildSearchTokenFilter(GROUP_SEARCH_TOKEN_FIELDS, query); +}