diff --git a/api/server/services/start/__tests__/migration.spec.js b/api/server/services/start/__tests__/migration.spec.js new file mode 100644 index 0000000000..ba4bfc32d8 --- /dev/null +++ b/api/server/services/start/__tests__/migration.spec.js @@ -0,0 +1,37 @@ +const mockWarnOnMissingSearchTokens = jest.fn(); +const mockCheckAgentPermissionsMigration = jest.fn(); +const mockCheckPromptPermissionsMigration = jest.fn(); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { error: jest.fn() }, + warnOnMissingSearchTokens: (...args) => mockWarnOnMissingSearchTokens(...args), +})); +jest.mock('@librechat/api', () => ({ + logAgentMigrationWarning: jest.fn(), + logPromptMigrationWarning: jest.fn(), + checkAgentPermissionsMigration: (...args) => mockCheckAgentPermissionsMigration(...args), + checkPromptPermissionsMigration: (...args) => mockCheckPromptPermissionsMigration(...args), +})); +jest.mock('~/models', () => ({ findRoleByIdentifier: jest.fn() })); + +const { checkMigrations } = require('../migration'); + +describe('checkMigrations', () => { + it('starts the search token probe while the permission checks are still running', async () => { + let finishAgentCheck; + mockCheckAgentPermissionsMigration.mockReturnValue( + new Promise((resolve) => { + finishAgentCheck = resolve; + }), + ); + mockCheckPromptPermissionsMigration.mockResolvedValue({}); + mockWarnOnMissingSearchTokens.mockResolvedValue(undefined); + + const done = checkMigrations(); + expect(mockWarnOnMissingSearchTokens).toHaveBeenCalledTimes(1); + + finishAgentCheck({}); + await done; + expect(mockCheckPromptPermissionsMigration).toHaveBeenCalledTimes(1); + }); +}); diff --git a/api/server/services/start/migration.js b/api/server/services/start/migration.js index 70f8300e08..d170590168 100644 --- a/api/server/services/start/migration.js +++ b/api/server/services/start/migration.js @@ -1,5 +1,5 @@ const mongoose = require('mongoose'); -const { logger } = require('@librechat/data-schemas'); +const { logger, warnOnMissingSearchTokens } = require('@librechat/data-schemas'); const { logAgentMigrationWarning, logPromptMigrationWarning, @@ -13,6 +13,8 @@ const { findRoleByIdentifier } = require('~/models'); * This runs at the end to ensure all systems are initialized */ async function checkMigrations() { + /** Started first so it overlaps the permission checks; it logs its own failures. */ + const searchTokens = warnOnMissingSearchTokens(mongoose.connection); try { const agentMigrationResult = await checkAgentPermissionsMigration({ mongoose, @@ -37,6 +39,7 @@ async function checkMigrations() { } catch (error) { logger.error('Failed to check prompt permissions migration:', error); } + await searchTokens; } module.exports = { diff --git a/config/migrate-search-tokens.js b/config/migrate-search-tokens.js new file mode 100644 index 0000000000..812e0e0338 --- /dev/null +++ b/config/migrate-search-tokens.js @@ -0,0 +1,30 @@ +require('dotenv').config(); +const mongoose = require('mongoose'); +const { backfillSearchTokens } = require('@librechat/data-schemas'); +const connect = require('./connect'); + +/** + * Backfills the word-prefix search tokens on users and groups saved before they existed. + * Idempotent; safe to run while the server is up. + * + * Usage: npm run migrate:search-tokens [-- --dry-run] [-- --batch-size=500] + */ +(async () => { + let exitCode = 0; + try { + await connect(); + const batchArg = process.argv.find((arg) => arg.startsWith('--batch-size=')); + const result = await backfillSearchTokens(mongoose.connection, { + dryRun: process.argv.includes('--dry-run'), + batchSize: batchArg ? parseInt(batchArg.split('=')[1], 10) || undefined : undefined, + }); + console.log(JSON.stringify(result, null, 2)); + } catch (error) { + console.error('Search token migration failed:', error); + exitCode = 1; + } finally { + await mongoose.disconnect(); + } + /** Exit explicitly: with Redis enabled, the cache clients opened on connect keep the process alive. */ + process.exit(exitCode); +})(); diff --git a/e2e/specs/mock/db.ts b/e2e/specs/mock/db.ts index 40127f7135..4a0492eb7a 100644 --- a/e2e/specs/mock/db.ts +++ b/e2e/specs/mock/db.ts @@ -16,7 +16,7 @@ function getRuntimeEnvPath(): string { ); } -function getMongoUri(): string { +export function getMongoUri(): string { try { const env = JSON.parse(fs.readFileSync(getRuntimeEnvPath(), 'utf8')) as { MONGO_URI?: string }; if (env.MONGO_URI) { diff --git a/e2e/specs/mock/scenarios/people-search-tokens.spec.ts b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts new file mode 100644 index 0000000000..5bef523e2b --- /dev/null +++ b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts @@ -0,0 +1,226 @@ +import path from 'path'; +import { promisify } from 'util'; +import { execFile } from 'child_process'; +import { expect, test } from '@playwright/test'; +import type { Page, Locator } from '@playwright/test'; +import { MOCK_ENDPOINTS, NEW_CHAT_PATH, getAccessToken, requestJson, uniqueName } from '../helpers'; +import { openAgentBuilder, uniqueAgentName, cleanupAgent } from '../agents.helpers'; +import { withMongo, getMongoUri } from '../db'; + +/** + * People search (`GET /api/permissions/search-principals`, the share dialog's + * picker) matches word prefixes over the derived `nameTokens` / `emailTokens` / + * `usernameTokens` arrays. These scenarios register users through the real + * auth route, so the schema middleware writes the tokens, and drive the + * backfill script (`npm run migrate:search-tokens`) against this run's database + * for documents a writer left without tokens or with stale ones. + * + * The primary e2e user is the first one registered in the run, so it is ADMIN + * and may search principals and share agents without seeding role permissions + * (see `principal-type-labels.spec.ts`). + */ + +const SEARCH_LABEL = 'Search for people or groups by name or email'; +const SEARCH_PATH = '/api/permissions/search-principals'; +const REPO_ROOT = path.resolve(__dirname, '../../../..'); + +type SearchResult = { name?: string; email?: string; type: string }; + +/** Letters only, so a suffix stays one search word. */ +function suffix(): string { + return Array.from({ length: 7 }, () => + String.fromCharCode(97 + Math.floor(Math.random() * 26)), + ).join(''); +} + +async function register(page: Page, name: string, email: string, username: string) { + const password = 'Search-tokens-pass-1'; + const response = await page.request.post('/api/auth/register', { + data: { name, email, username, password, confirm_password: password }, + }); + expect(response.ok(), await response.text()).toBeTruthy(); +} + +async function search(page: Page, token: string, q: string): Promise { + const body = await requestJson<{ results: SearchResult[] }>(page, { + path: `${SEARCH_PATH}?q=${encodeURIComponent(q)}&limit=20`, + token, + }); + return body.results; +} + +async function deleteUsers(emails: string[]): Promise { + await withMongo(async (db) => { + await db.collection('users').deleteMany({ email: { $in: emails } }); + }); +} + +async function openShareDialog(page: Page): Promise<{ dialog: Locator; agentId: string }> { + const agentName = uniqueAgentName('E2E People Search'); + const token = await getAccessToken(page); + const agent = await requestJson<{ id: string }>(page, { + path: '/api/agents', + token, + method: 'POST', + body: { name: agentName, provider: MOCK_ENDPOINTS[0].label, model: MOCK_ENDPOINTS[0].model }, + }); + const form = await openAgentBuilder(page); + await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); + await page.getByRole('option', { name: agentName, exact: true }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(agentName); + await page.getByRole('button', { name: `Share ${agentName}` }).click(); + const dialog = page.getByRole('dialog', { name: `Share ${agentName}` }); + await expect(dialog).toBeVisible(); + return { dialog, agentId: agent.id }; +} + +const execFileAsync = promisify(execFile); + +/** Async so a stuck script fails the test on its timeout instead of blocking the worker. */ +async function runBackfill(): Promise { + try { + await execFileAsync('node', ['config/migrate-search-tokens.js'], { + cwd: REPO_ROOT, + env: { ...process.env, MONGO_URI: getMongoUri() }, + timeout: 60000, + killSignal: 'SIGKILL', + }); + } catch (error) { + const { stdout = '', stderr = '' } = error as { stdout?: string; stderr?: string }; + throw new Error( + `Backfill failed: ${String(error)}\n${stdout.slice(-2000)}\n${stderr.slice(-2000)}`, + ); + } +} + +test.describe('people search over word-prefix tokens', () => { + test('@scenario:people-search-matches-word-prefixes the share dialog finds a user by the start of each name word, and not by a fragment inside a word', async ({ + page, + }) => { + test.setTimeout(120000); + const id = suffix(); + const user = { name: `Quill${id} Brightwater`, email: `${uniqueName('quill')}@search.test` }; + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await register(page, user.name, user.email, `quill${id}`); + + const opened = await openShareDialog(page); + agentId = opened.agentId; + const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL }); + await searchBox.fill(`brig quill${id.slice(0, 3)}`); + await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({ + timeout: 10000, + }); + + const token = await getAccessToken(page); + expect((await search(page, token, `ill${id}`)).map((r) => r.email)).not.toContain(user.email); + expect((await search(page, token, user.email.slice(0, 12))).map((r) => r.email)).toContain( + user.email, + ); + } finally { + await cleanupAgent(page, agentId); + await deleteUsers([user.email]); + } + }); + + test('@scenario:people-search-folds-accents-and-ranks-exact-first an unaccented query finds an accented name and ranks its exact match above a longer prefix match', async ({ + page, + }) => { + const id = suffix(); + const exact = { name: `Zoë${id}`, email: `${uniqueName('zoe-exact')}@search.test` }; + const prefix = { name: `Zoe${id}x Park`, email: `${uniqueName('zoe-prefix')}@search.test` }; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await register(page, prefix.name, prefix.email, `zoep${id}`); + await register(page, exact.name, exact.email, `zoee${id}`); + + const token = await getAccessToken(page); + const results = await search(page, token, `zoe${id}`); + expect(results.map((r) => r.name)).toEqual([exact.name, prefix.name]); + } finally { + await deleteUsers([exact.email, prefix.email]); + } + }); + + test('@scenario:backfill-repairs-missing-and-stale-tokens a user saved without tokens is still found, a user whose tokens went stale is found under the new name once the backfill runs', async ({ + page, + }) => { + test.setTimeout(120000); + const id = suffix(); + const legacy = { name: `Legacy${id} Person`, email: `${uniqueName('legacy')}@search.test` }; + const renamed = { name: `Renamed${id} Person`, email: `${uniqueName('renamed')}@search.test` }; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + const primary = await withMongo((db) => + db.collection('users').findOne({}, { sort: { createdAt: 1 } }), + ); + /** Raw writes, as an older server would leave them: one user with no + * tokens at all, one renamed without its tokens being rewritten. */ + await withMongo(async (db) => { + const now = new Date(); + await db.collection('users').insertMany([ + { ...legacy, tenantId: primary?.tenantId, createdAt: now, updatedAt: now }, + { + ...renamed, + tenantId: primary?.tenantId, + nameTokens: [`original${id}`, 'person'], + emailTokens: [renamed.email], + usernameTokens: [], + createdAt: now, + updatedAt: now, + }, + ]); + }); + + const token = await getAccessToken(page); + const emails = async (q: string) => (await search(page, token, q)).map((r) => r.email); + expect(await emails(`gacy${id}`)).toContain(legacy.email); + expect(await emails(`renamed${id}`)).not.toContain(renamed.email); + + await runBackfill(); + + expect(await emails(`renamed${id}`)).toContain(renamed.email); + expect(await emails(`original${id}`)).not.toContain(renamed.email); + expect(await emails(`legacy${id}`)).toContain(legacy.email); + const stored = await withMongo((db) => + db.collection('users').findOne({ email: legacy.email }), + ); + expect(stored?.nameTokens).toEqual([`legacy${id}`, 'person']); + } finally { + await deleteUsers([legacy.email, renamed.email]); + } + }); + + test('@scenario:people-picker-searches-once-per-pause typing a query character by character sends one search request, not one per keystroke', async ({ + page, + }) => { + test.setTimeout(120000); + const id = suffix(); + const user = { name: `Pause${id} Typist`, email: `${uniqueName('pause')}@search.test` }; + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await register(page, user.name, user.email, `pause${id}`); + const opened = await openShareDialog(page); + agentId = opened.agentId; + + const queries: string[] = []; + page.on('request', (request) => { + const url = new URL(request.url()); + if (url.pathname === SEARCH_PATH) { + queries.push(url.searchParams.get('q') ?? ''); + } + }); + const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL }); + await searchBox.pressSequentially(`pause${id}`, { delay: 40 }); + await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({ + timeout: 10000, + }); + expect(queries).toEqual([`pause${id}`]); + } finally { + await cleanupAgent(page, agentId); + await deleteUsers([user.email]); + } + }); +}); diff --git a/package.json b/package.json index 468915b789..72dde5ff7f 100644 --- a/package.json +++ b/package.json @@ -143,6 +143,8 @@ "migrate:tenant-indexes": "node config/migrate-tenant-indexes.js", "migrate:tenant-indexes:dry-run": "node config/migrate-tenant-indexes.js --dry-run", "migrate:terms-timestamp": "node config/migrate-terms-timestamp.js", + "migrate:search-tokens": "node config/migrate-search-tokens.js", + "migrate:search-tokens:dry-run": "node config/migrate-search-tokens.js --dry-run", "flush-cache": "node config/flush-cache.js", "migrate:agent-permissions:dry-run": "node config/migrate-agent-permissions.js --dry-run", "migrate:agent-permissions": "node config/migrate-agent-permissions.js", diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index 3c1aea1303..b3582c33f1 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -1,4 +1,5 @@ import { Types } from 'mongoose'; +import { buildUserSearchFilter } from '@librechat/data-schemas'; import { PrincipalType, SystemRoles } from 'librechat-data-provider'; import type { IUser, UserDeleteResult } from '@librechat/data-schemas'; import type { Response } from 'express'; @@ -205,20 +206,23 @@ describe('createAdminUsersHandlers', () => { expect(response.capped).toBe(true); }); - it('searches name, email, and username with anchored prefix regex', async () => { + it('searches the indexed word-prefix tokens of name, email, and username', async () => { const findUsers = jest.fn().mockResolvedValue([]); const deps = createDeps({ findUsers }); const handlers = createAdminUsersHandlers(deps); - const { req, res } = createReqRes({ query: { q: 'test' } }); + const { req, res } = createReqRes({ query: { q: 'Test' } }); await handlers.searchUsers(req, res); const filter = findUsers.mock.calls[0][0]; - expect(filter.$or).toHaveLength(3); - expect(filter.$or[0]).toHaveProperty('name'); - expect(filter.$or[1]).toHaveProperty('email'); - expect(filter.$or[2]).toHaveProperty('username'); - expect(filter.$or[0].name.source).toBe('^test'); + expect(filter).toEqual(buildUserSearchFilter('Test', { legacyPrefix: true })); + expect(filter.$or).toEqual( + expect.arrayContaining([ + { nameTokens: /^test/ }, + { emailTokens: /^test/ }, + { usernameTokens: /^test/ }, + ]), + ); }); it('projects username in the field selection', async () => { @@ -242,8 +246,25 @@ describe('createAdminUsersHandlers', () => { await handlers.searchUsers(req, res); const filter = findUsers.mock.calls[0][0]; - expect(filter.$or[0].name).toBeInstanceOf(RegExp); - expect(filter.$or[0].name.source).toBe('^test\\.user\\+1'); + expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^test\.user\+1/ }); + /** Documents awaiting the backfill keep the old anchored prefix match. */ + expect(filter.$and[0].$or).toContainEqual({ + nameTokens: { $exists: false }, + name: /^test\.user\+1/i, + }); + }); + + it('returns no users without querying when the query has nothing to match', async () => { + const findUsers = jest.fn(); + const deps = createDeps({ findUsers }); + const handlers = createAdminUsersHandlers(deps); + const { req, res, status, json } = createReqRes({ query: { q: '\u0301\u0301' } }); + + await handlers.searchUsers(req, res); + + expect(findUsers).not.toHaveBeenCalled(); + expect(status).toHaveBeenCalledWith(200); + expect(json).toHaveBeenCalledWith({ users: [], total: 0, capped: false }); }); it('returns 400 when query is missing', async () => { diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts index ae8ac19c9c..d28569e50a 100644 --- a/packages/api/src/admin/users.ts +++ b/packages/api/src/admin/users.ts @@ -1,7 +1,7 @@ import { Types } from 'mongoose'; import { randomUUID } from 'node:crypto'; import { PrincipalType, SystemRoles } from 'librechat-data-provider'; -import { logger, isValidObjectIdString } from '@librechat/data-schemas'; +import { logger, isValidObjectIdString, buildUserSearchFilter } from '@librechat/data-schemas'; import type { IUser, IConfig, @@ -197,14 +197,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { } const searchLimit = Math.min(Math.max(1, parseInt(limitStr, 10) || 20), 50); - const escaped = trimmed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const regex = new RegExp(`^${escaped}`, 'i'); - - const users = await findUsers( - { $or: [{ name: regex }, { email: regex }, { username: regex }] }, - '_id name email username avatar', - { limit: searchLimit, sort: { name: 1 } }, - ); + const filter = buildUserSearchFilter(trimmed, { legacyPrefix: true }); + const users = filter + ? await findUsers(filter, '_id name email username avatar', { + limit: searchLimit, + sort: { name: 1 }, + }) + : []; const results: AdminUserSearchResult[] = users.map((u) => ({ id: u._id?.toString() ?? '', diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 560d21ebdb..38b269be70 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -117,4 +117,7 @@ export { migrateTenantIndexes, dropSupersededPromptGroupIndexes, backfillMCPServerNormalizedNames, + backfillSearchTokens, + warnOnMissingSearchTokens, } from './migrations'; +export type { SearchTokenBackfillResult } from './migrations'; diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index d9318173d1..7bffe5e381 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -1842,6 +1842,35 @@ describe('User Methods - Database Tests', () => { expect(results.length).toBeGreaterThan(0); }); + test('should rank an accent-folded exact match above a prefix match', async () => { + await User.create([ + { name: 'Zoe Adams', email: 'zadams@test.com', username: 'zadams', provider: 'local' }, + { name: 'Zoë', email: 'zoe-umlaut@test.com', username: 'zumlaut', provider: 'local' }, + ]); + + const results = await methods.searchUsers({ searchPattern: 'zoe', limit: 1 }); + + expect(results).toHaveLength(1); + expect((results[0] as unknown as t.IUser).name).toBe('Zoë'); + }); + + test('should rank a reordered full-name match above a longer prefix match', async () => { + await User.create([ + { + name: 'Smith Johnson', + email: 'sjohnson@test.com', + username: 'sjohnson', + provider: 'local', + }, + { name: 'John Smith', email: 'jsmith@test.com', username: 'jsmith', provider: 'local' }, + ]); + + const results = await methods.searchUsers({ searchPattern: 'smith john', limit: 1 }); + + expect(results).toHaveLength(1); + expect((results[0] as unknown as t.IUser).name).toBe('John Smith'); + }); + test('should respect limit', async () => { const results = await methods.searchUsers({ searchPattern: 'example', limit: 2 }); @@ -1869,9 +1898,11 @@ describe('User Methods - Database Tests', () => { }); const results = await methods.searchUsers({ searchPattern: '.*' }); + expect(results).toEqual([]); - expect(results).toHaveLength(1); - expect((results[0] as unknown as t.IUser).name).toBe('Literal .* User'); + const literal = await methods.searchUsers({ searchPattern: 'literal' }); + expect(literal).toHaveLength(1); + expect((literal[0] as unknown as t.IUser).name).toBe('Literal .* User'); }); test('should handle invalid regex syntax as literal search text', async () => { diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index cc95028344..94aaddb6f1 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -16,8 +16,8 @@ import type { } from '~/types'; import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types'; import type { CacheStore } from '~/types'; +import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search'; import { evictAuthUserDocs } from '~/utils/eviction'; -import { escapeRegExp } from '~/utils/string'; import { signPayload } from '~/crypto'; import logger from '~/config/winston'; @@ -891,7 +891,7 @@ export function createUserMethods( } /** - * Search for users by pattern matching on name, email, or username (case-insensitive) + * Search for users by word prefix of name, email, or username (see `buildUserSearchFilter`) * @param searchPattern - The pattern to search for * @param limit - Maximum number of results to return * @param fieldsToSelect - The fields to include or exclude in the returned documents @@ -975,12 +975,13 @@ export function createUserMethods( } const trimmedPattern = searchPattern.trim(); - const regex = new RegExp(escapeRegExp(trimmedPattern), 'i'); + const filter = buildUserSearchFilter(trimmedPattern); + if (!filter) { + return []; + } const User = mongoose.models.User; - const query = User.find({ - $or: [{ email: regex }, { name: regex }, { username: regex }], - }).limit(limit * 2); // Get more results to allow for relevance sorting + const query = User.find(filter).limit(limit * 2); // Get more results to allow for relevance sorting if (fieldsToSelect) { query.select(fieldsToSelect); @@ -988,41 +989,10 @@ export function createUserMethods( const users = await query.lean(); - // Score results by relevance - const startsWithPattern = trimmedPattern.toLowerCase(); - - const scoredUsers = users.map((user) => { - const searchableFields = [user.name, user.email, user.username].filter( - (field): field is string => typeof field === 'string' && field.length > 0, - ); - let maxScore = 0; - - for (const field of searchableFields) { - const fieldLower = field.toLowerCase(); - let score = 0; - - // Exact match gets highest score - if (fieldLower === startsWithPattern) { - score = 100; - } - // Starts with query gets high score - else if (fieldLower.startsWith(startsWithPattern)) { - score = 80; - } - // Contains query gets medium score - else if (fieldLower.includes(startsWithPattern)) { - score = 50; - } - // Default score for database match - else { - score = 10; - } - - maxScore = Math.max(maxScore, score); - } - - return { ...user, _searchScore: maxScore }; - }); + const scoredUsers = users.map((user) => ({ + ...user, + _searchScore: scoreSearchMatch([user.name, user.email, user.username], trimmedPattern), + })); /** Top results sorted by relevance */ return scoredUsers diff --git a/packages/data-schemas/src/methods/userGroup.spec.ts b/packages/data-schemas/src/methods/userGroup.spec.ts index db3212e596..9c4a3a8ce0 100644 --- a/packages/data-schemas/src/methods/userGroup.spec.ts +++ b/packages/data-schemas/src/methods/userGroup.spec.ts @@ -117,16 +117,18 @@ describe('userGroup methods', () => { expect(results[0].name).toBe('Design'); }); - it('matches on description field', async () => { + it('does not match on the description field (search covers name and email)', async () => { const results = await methods.findGroupsByNamePattern('Eng team'); - expect(results).toHaveLength(1); - expect(results[0].name).toBe('Engineering'); + expect(results).toEqual([]); }); it('treats regex metacharacters as literal text', async () => { const results = await methods.findGroupsByNamePattern('.*'); - expect(results).toHaveLength(1); - expect(results[0].name).toBe('Literal .* Group'); + expect(results).toEqual([]); + + const literal = await methods.findGroupsByNamePattern('literal'); + expect(literal).toHaveLength(1); + expect(literal[0].name).toBe('Literal .* Group'); }); it('filters by source when provided', async () => { @@ -1245,6 +1247,14 @@ describe('userGroup methods', () => { expect(score).toBe(50); }); + it('scores accent-folded matches like their plain spelling', () => { + const score = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'Zoë', source: 'local' }, + 'zoe', + ); + expect(score).toBe(100); + }); + it('returns 10 (default) when no substring or exact match', () => { const score = methods.calculateRelevanceScore( { type: PrincipalType.USER, name: 'bob', source: 'local' }, @@ -1267,17 +1277,51 @@ describe('userGroup methods', () => { expect(score).toBe(100); }); - it('checks description for GROUP type', () => { - const score = methods.calculateRelevanceScore( + it('ignores description for GROUP type, which search no longer matches', () => { + const weak = methods.calculateRelevanceScore( { type: PrincipalType.GROUP, - name: 'other', - description: 'alice team', + name: 'Corporate Platform', + description: 'plat', source: 'local', }, - 'alice', + 'plat', ); - expect(score).toBe(80); + const strong = methods.calculateRelevanceScore( + { type: PrincipalType.GROUP, name: 'Platform Team', source: 'local' }, + 'plat', + ); + expect(weak).toBe(50); + expect(strong).toBeGreaterThan(weak); + }); + + it('scores a name equal to the query up to separators as an exact match', () => { + const score = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'Mary-Jane', source: 'local' }, + 'mary jane', + ); + expect(score).toBe(100); + }); + + it('scores a reordered full-name query as exact, above a longer prefix match', () => { + const exact = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'John Smith', source: 'local' }, + 'smith john', + ); + const prefix = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'Smith Johnson', source: 'local' }, + 'smith john', + ); + expect(exact).toBe(100); + expect(prefix).toBe(80); + }); + + it('scores query words matched across fields like a contains match', () => { + const score = methods.calculateRelevanceScore( + { type: PrincipalType.USER, name: 'John Doe', email: 'jd@gmail.com', source: 'local' }, + 'john gmail', + ); + expect(score).toBe(50); }); it('picks the highest score across multiple fields', () => { @@ -1402,9 +1446,11 @@ describe('userGroup methods', () => { }); const results = await methods.searchPrincipals('.*', 10, [PrincipalType.USER]); + expect(results).toEqual([]); - expect(results).toHaveLength(1); - expect(results[0].name).toBe('Literal .* User'); + const literal = await methods.searchPrincipals('literal', 10, [PrincipalType.USER]); + expect(literal).toHaveLength(1); + expect(literal[0].name).toBe('Literal .* User'); }); it('handles invalid regex syntax as literal search text', async () => { diff --git a/packages/data-schemas/src/methods/userGroup.ts b/packages/data-schemas/src/methods/userGroup.ts index 451f8e97c4..feefa83460 100644 --- a/packages/data-schemas/src/methods/userGroup.ts +++ b/packages/data-schemas/src/methods/userGroup.ts @@ -4,6 +4,7 @@ import { CacheKeys, PrincipalType, SystemRoles } from 'librechat-data-provider'; import type { TPrincipalSearchResult } from 'librechat-data-provider'; import type { Model, ClientSession, FilterQuery } from 'mongoose'; import type { CacheStore, IGroup, IRole, IUser } from '~/types'; +import { buildUserSearchFilter, scoreSearchMatch, buildGroupSearchFilter } from '~/utils/search'; import { isValidObjectIdString } from '~/utils/objectId'; import { scopedCacheKey } from '~/config/tenantContext'; import { escapeRegExp } from '~/utils/string'; @@ -607,7 +608,7 @@ export function createUserGroupMethods( } /** - * Find groups by name pattern (case-insensitive partial match) + * Find groups by word prefix of name or email (see `buildGroupSearchFilter`) * @param namePattern - The name pattern to search for * @param source - Optional source filter ('entra', 'local', or null for all) * @param limit - Maximum number of results to return @@ -621,10 +622,11 @@ export function createUserGroupMethods( session?: ClientSession, ): Promise { const Group = mongoose.models.Group as Model; - const regex = new RegExp(escapeRegExp(namePattern), 'i'); - const query: Record = { - $or: [{ name: regex }, { email: regex }, { description: regex }], - }; + const filter = buildGroupSearchFilter(namePattern); + if (!filter) { + return []; + } + const query: Record = { ...filter }; if (source) { query.source = source; @@ -869,39 +871,12 @@ export function createUserGroupMethods( * @returns Relevance score (0-100) */ function calculateRelevanceScore(item: TPrincipalSearchResult, searchPattern: string): number { - const normalizedPattern = searchPattern.toLowerCase(); - - /** Get searchable text based on type */ - const searchableFields = + /** The fields each search matches on; a group's description is not one of them */ + const values = item.type === PrincipalType.USER - ? [item.name, item.email, item.username].filter(Boolean) - : [item.name, item.email, item.description].filter(Boolean); - - let maxScore = 0; - - for (const field of searchableFields) { - if (!field) continue; - const fieldLower = field.toLowerCase(); - let score = 0; - - /** Exact match gets highest score */ - if (fieldLower === normalizedPattern) { - score = 100; - } else if (fieldLower.startsWith(normalizedPattern)) { - /** Starts with query gets high score */ - score = 80; - } else if (fieldLower.includes(normalizedPattern)) { - /** Contains query gets medium score */ - score = 50; - } else { - /** Default score for database match */ - score = 10; - } - - maxScore = Math.max(maxScore, score); - } - - return maxScore; + ? [item.name, item.email, item.username] + : [item.name, item.email]; + return scoreSearchMatch(values, searchPattern); } /** @@ -970,7 +945,8 @@ export function createUserGroupMethods( } /** - * Search for principals (users and groups) by pattern matching on name/email + * Search for principals: users and groups by word prefix (see `buildSearchTokenFilter`), + * roles by name * Returns combined results in TPrincipalSearchResult format without sorting * @param searchPattern - The pattern to search for * @param limitPerType - Maximum number of results to return @@ -992,17 +968,11 @@ export function createUserGroupMethods( const escapedPattern = escapeRegExp(trimmedPattern); const promises: Promise[] = []; - if (!typeFilter || typeFilter.includes(PrincipalType.USER)) { - /** Note: searchUsers is imported from ~/models and needs to be passed in or implemented */ + const userFilter = buildUserSearchFilter(trimmedPattern); + if (userFilter && (!typeFilter || typeFilter.includes(PrincipalType.USER))) { const userFields = 'name email username avatar provider idOnTheSource role'; - /** For now, we'll use a direct query instead of searchUsers */ const User = mongoose.models.User as Model; - const regex = new RegExp(escapedPattern, 'i'); - const userQuery = User.find({ - $or: [{ name: regex }, { email: regex }, { username: regex }], - }) - .select(userFields) - .limit(limitPerType); + const userQuery = User.find(userFilter).select(userFields).limit(limitPerType); if (session) { userQuery.session(session); diff --git a/packages/data-schemas/src/migrations/index.ts b/packages/data-schemas/src/migrations/index.ts index c3ba619181..ed2699d8a4 100644 --- a/packages/data-schemas/src/migrations/index.ts +++ b/packages/data-schemas/src/migrations/index.ts @@ -2,3 +2,5 @@ export { dropSupersededTenantIndexes, migrateTenantIndexes } from './tenantIndex export { dropSupersededPromptGroupIndexes } from './promptGroupIndexes'; export { createMCPAuthorityLookupIndexes } from './mcpAuthorityIndexes'; export { MCPServerNameMigrationError, backfillMCPServerNormalizedNames } from './mcpServerNames'; +export { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens'; +export type { SearchTokenBackfillResult } from './searchTokens'; diff --git a/packages/data-schemas/src/migrations/searchTokens.spec.ts b/packages/data-schemas/src/migrations/searchTokens.spec.ts new file mode 100644 index 0000000000..38c62f81f5 --- /dev/null +++ b/packages/data-schemas/src/migrations/searchTokens.spec.ts @@ -0,0 +1,247 @@ +import mongoose from 'mongoose'; +import { Collection } from 'mongodb'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { Document, FindOptions } from 'mongodb'; +import { + searchTokenIndexes, + USER_SEARCH_TOKEN_FIELDS, + GROUP_SEARCH_TOKEN_FIELDS, +} from '~/utils/search'; +import { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens'; +import logger from '~/config/winston'; + +jest.mock('~/config/winston', () => ({ + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), +})); + +let mongoServer: MongoMemoryServer; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(async () => { + await mongoose.connection.dropDatabase(); + jest.clearAllMocks(); +}); + +const users = () => mongoose.connection.db!.collection('users'); +const groups = () => mongoose.connection.db!.collection('groups'); + +describe('backfillSearchTokens', () => { + it('backfills users and groups across tenants in batches and is idempotent', async () => { + await users().insertMany([ + { name: 'Ana Lima', email: 'ana@x.io', username: 'ana', tenantId: 'tenant-a' }, + { name: 'Ben Ode', email: 'ben@y.io', tenantId: 'tenant-b' }, + { email: 'noname@z.io' }, + { + name: 'Already Done', + email: 'done@z.io', + nameTokens: ['already', 'done'], + emailTokens: ['done@z.io', 'done', 'z', 'io'], + usernameTokens: [], + }, + ]); + await groups().insertMany([{ name: 'Core Team', source: 'local' }]); + + const result = await backfillSearchTokens(mongoose.connection, { batchSize: 2 }); + expect(result).toEqual({ + pending: { users: 3, groups: 1 }, + updated: { users: 3, groups: 1 }, + }); + + const ana = await users().findOne({ email: 'ana@x.io' }); + expect(ana).toMatchObject({ + tenantId: 'tenant-a', + nameTokens: ['ana', 'lima'], + emailTokens: ['ana@x.io', 'ana', 'x', 'io'], + usernameTokens: ['ana'], + }); + expect(await users().findOne({ email: 'noname@z.io' })).toMatchObject({ + nameTokens: [], + usernameTokens: [], + }); + expect(await groups().findOne({ name: 'Core Team' })).toMatchObject({ + nameTokens: ['core', 'team'], + emailTokens: [], + }); + + await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({ + pending: { users: 0, groups: 0 }, + updated: { users: 0, groups: 0 }, + }); + }); + + it('completes documents that a partial update tokenized for only one field', async () => { + await users().insertOne({ name: 'Part Ial', email: 'p@x.io', emailTokens: ['p@x.io'] }); + await backfillSearchTokens(mongoose.connection); + expect(await users().findOne({ email: 'p@x.io' })).toMatchObject({ + nameTokens: ['part', 'ial'], + emailTokens: ['p@x.io', 'p', 'x', 'io'], + }); + }); + + it('repairs tokens left stale by a writer that did not maintain them', async () => { + await users().insertOne({ + name: 'New Name', + email: 'stale@x.io', + nameTokens: ['old', 'name'], + emailTokens: ['stale@x.io', 'stale', 'x', 'io'], + usernameTokens: [], + }); + await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({ + pending: { users: 1, groups: 0 }, + updated: { users: 0, groups: 0 }, + }); + await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({ + pending: { users: 1, groups: 0 }, + updated: { users: 1, groups: 0 }, + }); + expect(await users().findOne({ email: 'stale@x.io' })).toMatchObject({ + nameTokens: ['new', 'name'], + }); + }); + + it('creates the token indexes when the schema indexes were never built', async () => { + await users().insertOne({ name: 'No Index', email: 'noindex@x.io' }); + await groups().insertOne({ name: 'Plain Group' }); + await backfillSearchTokens(mongoose.connection); + const userIndexes = (await users().indexes()).map((index) => index.name); + const groupIndexes = (await groups().indexes()).map((index) => index.name); + expect(userIndexes).toEqual( + expect.arrayContaining([ + 'nameTokens_1_tenantId_1', + 'emailTokens_1_tenantId_1', + 'usernameTokens_1_tenantId_1', + 'tenantId_1_nameTokens_1', + 'tenantId_1_emailTokens_1', + 'tenantId_1_usernameTokens_1', + ]), + ); + expect(groupIndexes).toEqual( + expect.arrayContaining([ + 'nameTokens_1_tenantId_1', + 'emailTokens_1_tenantId_1', + 'tenantId_1_nameTokens_1', + 'tenantId_1_emailTokens_1', + ]), + ); + }); + + it('writes nothing on a dry run', async () => { + await users().insertOne({ name: 'Dry Run', email: 'dry@x.io' }); + await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({ + pending: { users: 1, groups: 0 }, + updated: { users: 0, groups: 0 }, + }); + expect(await users().findOne({ email: 'dry@x.io' })).not.toHaveProperty('nameTokens'); + }); +}); + +describe('warnOnMissingSearchTokens', () => { + it('warns only while documents lack tokens', async () => { + await users().insertOne({ name: 'Old User', email: 'old@x.io' }); + await groups().insertOne({ name: 'Old Group' }); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users and groups')); + + jest.clearAllMocks(); + await backfillSearchTokens(mongoose.connection); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + /** Runs the startup check and explains the `users` probe it actually issued. */ + const explainUserProbe = async () => { + const findOne = jest.spyOn(Collection.prototype, 'findOne'); + try { + await warnOnMissingSearchTokens(mongoose.connection); + const call = findOne.mock.contexts.findIndex( + (collection) => collection.collectionName === 'users', + ); + const [filter, options] = findOne.mock.calls[call] as [Document, FindOptions]; + expect(options.maxTimeMS).toBeGreaterThan(0); + return await users().find(filter).limit(1).explain('executionStats'); + } finally { + findOne.mockRestore(); + } + }; + + it('warns without probing documents while the token indexes do not exist', async () => { + await users().insertMany( + Array.from({ length: 200 }, (_, i) => ({ name: `Old ${i}`, email: `old${i}@x.io` })), + ); + const findOne = jest.spyOn(Collection.prototype, 'findOne'); + try { + await warnOnMissingSearchTokens(mongoose.connection); + const probed = findOne.mock.contexts.some( + (collection) => collection.collectionName === 'users', + ); + expect(probed).toBe(false); + } finally { + findOne.mockRestore(); + } + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users')); + }); + + it('examines no documents once the backfill has built the indexes and filled every token', async () => { + await users().insertMany( + Array.from({ length: 50 }, (_, i) => ({ name: `User ${i}`, email: `u${i}@x.io` })), + ); + await backfillSearchTokens(mongoose.connection); + jest.clearAllMocks(); + const explain = await explainUserProbe(); + expect(JSON.stringify(explain.queryPlanner.winningPlan)).not.toContain('COLLSCAN'); + expect(explain.executionStats.totalDocsExamined).toBe(0); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('warns when every document has tokens but the token indexes were never built', async () => { + await users().insertOne({ + name: 'Fresh User', + email: 'fresh@x.io', + nameTokens: ['fresh', 'user'], + emailTokens: ['fresh@x.io', 'fresh', 'x', 'io'], + usernameTokens: [], + }); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users')); + + jest.clearAllMocks(); + await backfillSearchTokens(mongoose.connection); + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('stays quiet on a database with no users or groups yet', async () => { + await warnOnMissingSearchTokens(mongoose.connection); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('logs a failed probe instead of failing startup', async () => { + /** Every token index exists, so the check reaches the probe, which then fails. */ + const indexes = [...USER_SEARCH_TOKEN_FIELDS, ...GROUP_SEARCH_TOKEN_FIELDS] + .flatMap(searchTokenIndexes) + .map((key) => ({ key })); + const findOne = jest.fn(() => Promise.reject(new Error('down'))); + const broken = { + db: { collection: () => ({ indexes: async () => indexes, findOne }) }, + }; + await expect( + warnOnMissingSearchTokens(broken as unknown as typeof mongoose.connection), + ).resolves.toBeUndefined(); + expect(findOne).toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/data-schemas/src/migrations/searchTokens.ts b/packages/data-schemas/src/migrations/searchTokens.ts new file mode 100644 index 0000000000..42d668883a --- /dev/null +++ b/packages/data-schemas/src/migrations/searchTokens.ts @@ -0,0 +1,178 @@ +import type { AnyBulkWriteOperation, Document } from 'mongodb'; +import type { Connection } from 'mongoose'; +import type { SearchTokenField } from '~/utils/search'; +import { + searchTokenIndexes, + computeSearchTokenSet, + USER_SEARCH_TOKEN_FIELDS, + GROUP_SEARCH_TOKEN_FIELDS, +} from '~/utils/search'; +import { buildIndexWithRetry } from '~/utils/retry'; +import logger from '~/config/winston'; + +const DEFAULT_BATCH_SIZE = 500; +/** Upper bound on the startup probe, in case it runs before the token indexes exist. */ +const STARTUP_PROBE_MAX_TIME_MS = 5_000; + +const SEARCH_TOKEN_COLLECTIONS: ReadonlyArray<{ + name: string; + fields: readonly SearchTokenField[]; +}> = [ + { name: 'users', fields: USER_SEARCH_TOKEN_FIELDS }, + { name: 'groups', fields: GROUP_SEARCH_TOKEN_FIELDS }, +]; + +export interface SearchTokenBackfillResult { + /** Documents whose token fields are missing or no longer match their source, per collection. */ + pending: Record; + /** Documents written, per collection; zero on a dry run. */ + updated: Record; +} + +/** Index-bound: each branch seeks one token index on its missing-key bounds. */ +function missingTokens(fields: readonly SearchTokenField[]): Document { + return { $or: fields.map((field) => ({ [field.tokens]: { $exists: false } })) }; +} + +function sameTokens(stored: unknown, expected: string[]): boolean { + return ( + Array.isArray(stored) && + stored.length === expected.length && + stored.every((token, index) => token === expected[index]) + ); +} + +/** + * Brings the search-token arrays on users and groups in line with their source + * fields: fills documents saved before the tokens existed and repairs tokens + * left stale by a writer that did not maintain them (an older server during a + * rolling deploy, a raw collection write). Every document is read and only the + * ones whose tokens differ are written, so it is idempotent and resumable; run + * it again once a rollout completes. Each write is guarded by the source values + * it was computed from, so a concurrent rename (whose own update already wrote + * fresh tokens) is skipped. Runs across all tenants on the raw collections; the + * `_id` filter keeps every write on its own document, and `tenantId` is never + * written. + */ +export async function backfillSearchTokens( + connection: Connection, + options: { batchSize?: number; dryRun?: boolean } = {}, +): Promise { + const batchSize = Math.max(1, options.batchSize ?? DEFAULT_BATCH_SIZE); + const result: SearchTokenBackfillResult = { pending: {}, updated: {} }; + + for (const { name, fields } of SEARCH_TOKEN_COLLECTIONS) { + const collection = connection.db!.collection(name); + if (!options.dryRun) { + /** Deployments running with `MONGO_AUTO_INDEX` off never build schema indexes, and + * without these the token filters would scan the collection. Same specs and default + * names as the schema declaration, so this is a no-op where Mongoose already built them. */ + for (const field of fields) { + for (const index of searchTokenIndexes(field)) { + await buildIndexWithRetry( + () => collection.createIndex(index), + `createIndex(${name}.${Object.keys(index).join('_')})`, + ); + } + } + } + result.pending[name] = 0; + result.updated[name] = 0; + + const projection = Object.fromEntries( + fields.flatMap((field) => [ + [field.source, 1], + [field.tokens, 1], + ]), + ); + let batch: AnyBulkWriteOperation[] = []; + const flush = async () => { + if (batch.length === 0) { + return; + } + // eslint-disable-next-line no-restricted-syntax -- offline all-tenant migration; `_id` filters keep each write on its own document + const written = await collection.bulkWrite(batch, { ordered: false }); + result.updated[name] += written.modifiedCount; + batch = []; + }; + + for await (const doc of collection.find({}, { projection })) { + const expected = computeSearchTokenSet(fields, doc); + if (fields.every((field) => sameTokens(doc[field.tokens], expected[field.tokens]))) { + continue; + } + result.pending[name] += 1; + if (options.dryRun) { + continue; + } + const guard = Object.fromEntries( + fields.map((field) => [field.source, doc[field.source] ?? null]), + ); + batch.push({ updateOne: { filter: { _id: doc._id, ...guard }, update: { $set: expected } } }); + if (batch.length >= batchSize) { + await flush(); + } + } + await flush(); + if (!options.dryRun) { + logger.info( + `[SearchTokenMigration] ${name}: ${result.updated[name]} of ${result.pending[name]} documents updated`, + ); + } + } + return result; +} + +/** + * Logs a startup warning when users or groups still lack search tokens. Those + * documents stay findable through the slower unindexed fallback until + * `npm run migrate:search-tokens` runs. A best-effort diagnostic: a failed + * check is logged and never blocks startup. + * + * Also warns when the token indexes are missing: with `MONGO_AUTO_INDEX` off, + * a database whose documents all received tokens from the schema middleware + * still never builds them until the backfill runs. + * + * The indexes are checked first, so the probe for a token-less document only + * runs once they exist and it can seek them; `maxTimeMS` bounds it regardless. + */ +export async function warnOnMissingSearchTokens(connection: Connection): Promise { + try { + const pending = await Promise.all( + SEARCH_TOKEN_COLLECTIONS.map(async ({ name, fields }) => { + const collection = connection.db!.collection(name); + const indexes = await collection.indexes().catch((error: { codeName?: string }) => { + /** No collection yet: nothing to search, nothing to migrate. */ + if (error?.codeName === 'NamespaceNotFound') { + return null; + } + throw error; + }); + if (indexes == null) { + return null; + } + const built = new Set(indexes.map((index) => JSON.stringify(index.key))); + const unindexed = fields.some((field) => + searchTokenIndexes(field).some((index) => !built.has(JSON.stringify(index))), + ); + /** Without the indexes the probe below would scan; the warning is already due. */ + if (unindexed) { + return name; + } + const unmigrated = await collection.findOne(missingTokens(fields), { + projection: { _id: 1 }, + maxTimeMS: STARTUP_PROBE_MAX_TIME_MS, + }); + return unmigrated != null ? name : null; + }), + ); + const names = pending.filter((name): name is string => name != null); + if (names.length > 0) { + logger.warn( + `[SearchTokenMigration] Some ${names.join(' and ')} lack search tokens or their indexes; people search scans the collection until you run: npm run migrate:search-tokens`, + ); + } + } catch (error) { + logger.error('[SearchTokenMigration] Failed to check search token migration:', error); + } +} diff --git a/packages/data-schemas/src/models/plugins/searchTokens.spec.ts b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts new file mode 100644 index 0000000000..edd1c6154d --- /dev/null +++ b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts @@ -0,0 +1,318 @@ +import mongoose from 'mongoose'; +import { PrincipalType } from 'librechat-data-provider'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type * as t from '~/types'; +import { createUserGroupMethods } from '~/methods/userGroup'; +import { tenantStorage } from '~/config/tenantContext'; +import { buildUserSearchFilter } from '~/utils/search'; +import { createModels } from '~/models'; + +jest.mock('~/config/winston', () => ({ + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), +})); + +const TOKENS = '+nameTokens +emailTokens +usernameTokens'; + +let mongoServer: MongoMemoryServer; +let User: mongoose.Model; +let Group: mongoose.Model; +let methods: ReturnType; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + const models = createModels(mongoose); + User = models.User; + Group = models.Group as mongoose.Model; + methods = createUserGroupMethods(mongoose); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(async () => { + await mongoose.connection.dropDatabase(); +}); + +const user = (name: string, email: string, username = '', extra: Partial = {}) => ({ + name, + email, + username, + provider: 'local', + ...extra, +}); + +async function tokensOf(id: unknown) { + return User.findById(id).select(TOKENS).lean(); +} + +async function userNames(query: string, limit = 10): Promise { + const results = await methods.searchPrincipals(query, limit, [PrincipalType.USER]); + return results.map((r) => r.name ?? '').sort(); +} + +describe('search token write paths', () => { + it('derives tokens on create and on save of a changed name', async () => { + const doc = await User.create(user('José Álvarez', 'Jose.Alvarez@Example.org', 'jalvarez')); + expect(await tokensOf(doc._id)).toMatchObject({ + nameTokens: ['jose', 'alvarez'], + emailTokens: ['jose.alvarez@example.org', 'jose', 'alvarez', 'example', 'org'], + usernameTokens: ['jalvarez'], + }); + + doc.name = 'Joe Bloggs'; + await doc.save(); + const saved = await tokensOf(doc._id); + expect(saved?.nameTokens).toEqual(['joe', 'bloggs']); + expect(saved?.usernameTokens).toEqual(['jalvarez']); + }); + + it('updates tokens through findOneAndUpdate, findByIdAndUpdate, updateOne and updateMany', async () => { + const [a, b] = await User.create([user('Ann A', 'a@x.io'), user('Ben B', 'b@x.io')]); + + await User.findOneAndUpdate({ _id: a._id }, { $set: { name: 'Carla Diaz' } }); + expect((await tokensOf(a._id))?.nameTokens).toEqual(['carla', 'diaz']); + + await User.findByIdAndUpdate(a._id, { email: 'carla@new.dev' }); + expect((await tokensOf(a._id))?.emailTokens).toEqual(['carla@new.dev', 'carla', 'new', 'dev']); + + await User.updateOne({ _id: b._id }, { $set: { username: 'ben_b' } }); + expect((await tokensOf(b._id))?.usernameTokens).toEqual(['ben_b', 'ben', 'b']); + + await User.updateMany({}, { $unset: { name: '' } }); + expect((await tokensOf(a._id))?.nameTokens).toEqual([]); + expect((await tokensOf(b._id))?.nameTokens).toEqual([]); + }); + + it('sets tokens on upsert inserts and insertMany', async () => { + await User.updateOne( + { email: 'new@x.io' }, + { $setOnInsert: { name: 'New Person', email: 'new@x.io', provider: 'local' } }, + { upsert: true }, + ); + const upserted = await User.findOne({ email: 'new@x.io' }).select(TOKENS).lean(); + expect(upserted?.nameTokens).toEqual(['new', 'person']); + expect(upserted?.usernameTokens).toEqual([]); + + await User.updateOne( + { email: 'filter@x.io' }, + { $set: { name: 'From Filter', provider: 'local' } }, + { upsert: true }, + ); + const fromFilter = await User.findOne({ email: 'filter@x.io' }).select(TOKENS).lean(); + expect(fromFilter?.emailTokens).toEqual(['filter@x.io', 'filter', 'x', 'io']); + + await User.insertMany([user('Many One', 'one@x.io', 'm1')]); + const inserted = await User.findOne({ email: 'one@x.io' }).select(TOKENS).lean(); + expect(inserted?.nameTokens).toEqual(['many', 'one']); + expect(inserted?.usernameTokens).toEqual(['m1']); + }); + + it('keeps tokens out of default reads', async () => { + const doc = await User.create(user('Hidden Tokens', 'h@x.io')); + const plain = await User.findById(doc._id).lean(); + expect(plain).not.toHaveProperty('nameTokens'); + }); + + it('updates group tokens through create, upsertGroupByExternalId and updateGroupById', async () => { + const group = await methods.createGroup({ name: 'Core Platform', source: 'local' }); + const read = () => + Group.findById(group._id).select('+nameTokens +emailTokens').lean(); + expect((await read())?.nameTokens).toEqual(['core', 'platform']); + + await methods.updateGroupById(group._id, { name: 'Data Science', email: 'ds@corp.io' }); + expect(await read()).toMatchObject({ + nameTokens: ['data', 'science'], + emailTokens: ['ds@corp.io', 'ds', 'corp', 'io'], + }); + + const entra = await methods.upsertGroupByExternalId('ext-1', 'entra', { name: 'Sales EMEA' }); + const entraTokens = await Group.findById(entra?._id).select('+nameTokens').lean(); + expect(entraTokens?.nameTokens).toEqual(['sales', 'emea']); + expect(await Group.countDocuments({ _id: entra?._id, emailTokens: { $exists: true } })).toBe(1); + }); + + it('keeps tokens when an update assigns undefined to a source field', async () => { + const doc = await User.create(user('Kept Name', 'kept@x.io')); + await User.findByIdAndUpdate(doc._id, { $set: { name: undefined, role: 'ADMIN' } }); + const stored = await User.findById(doc._id).select(`name ${TOKENS}`).lean(); + expect(stored?.name).toBe('Kept Name'); + expect(stored?.nameTokens).toEqual(['kept', 'name']); + }); +}); + +describe('searchPrincipals word-prefix search', () => { + beforeEach(async () => { + await User.create([ + user('John Smith', 'john.smith@acme.com', 'jsmith'), + user('Johanna Müller', 'jm@globex.io', 'hanna_m'), + user('Bob Johnson', 'bob@initech.net', 'bobby'), + user('Zoë Ng', 'zoe@acme.com', 'zng'), + ]); + }); + + it('matches the prefix of a first name, a last name, an email word, a domain and a username', async () => { + expect(await userNames('joh')).toEqual(['Bob Johnson', 'Johanna Müller', 'John Smith']); + expect(await userNames('smi')).toEqual(['John Smith']); + expect(await userNames('mull')).toEqual(['Johanna Müller']); + expect(await userNames('acme')).toEqual(['John Smith', 'Zoë Ng']); + expect(await userNames('hanna')).toEqual(['Johanna Müller']); + expect(await userNames('john.smith@ac')).toEqual(['John Smith']); + }); + + it('requires every query token, across fields', async () => { + expect(await userNames('john smi')).toEqual(['John Smith']); + expect(await userNames('smith john')).toEqual(['John Smith']); + expect(await userNames('jo acme')).toEqual(['John Smith']); + expect(await userNames('john globex')).toEqual([]); + }); + + it('is case- and accent-insensitive', async () => { + expect(await userNames('ZOE')).toEqual(['Zoë Ng']); + expect(await userNames('MÜLLER')).toEqual(['Johanna Müller']); + }); + + it('does not match inside a word (intended: only word prefixes are index-bound)', async () => { + expect(await userNames('ohn')).toEqual([]); + expect(await userNames('mith')).toEqual([]); + }); + + it('respects the limit', async () => { + expect(await userNames('jo', 2)).toHaveLength(2); + }); + + it('finds documents written before tokens existed through the substring fallback', async () => { + await User.collection.insertOne({ name: 'Legacy Person', email: 'legacy@old.io' }); + expect(await userNames('gacy')).toEqual(['Legacy Person']); + }); + + it('stays inside the active tenant', async () => { + await tenantStorage.run({ tenantId: 'tenant-a' }, () => + User.create(user('John Tenant', 'john@tenant-a.io')), + ); + await tenantStorage.run({ tenantId: 'tenant-b' }, () => + User.create(user('John Other', 'john@tenant-b.io')), + ); + const names = await tenantStorage.run({ tenantId: 'tenant-a' }, () => userNames('john')); + expect(names).toEqual(['John Tenant']); + }); + + it('finds groups by name and email word prefix', async () => { + await Group.create([ + { name: 'Platform Team', source: 'local', email: 'platform@corp.io' }, + { name: 'Design', source: 'local', description: 'Platform design' }, + ]); + const groups = await methods.searchPrincipals('plat', 10, [PrincipalType.GROUP]); + expect(groups.map((g) => g.name)).toEqual(['Platform Team']); + }); +}); + +describe('search token serialization', () => { + it('keeps token fields out of the JSON and object output of a freshly created document', async () => { + const [group] = await Group.create([{ name: 'Platform Team', source: 'local' }]); + const created = await User.create(user('Ana Lima', 'ana@x.io', 'ana')); + for (const doc of [group, created]) { + expect(doc.get('nameTokens')).toBeDefined(); + expect(doc.toJSON()).not.toHaveProperty('nameTokens'); + expect(doc.toObject()).not.toHaveProperty('nameTokens'); + expect(doc.toObject()).not.toHaveProperty('emailTokens'); + } + const stored = await Group.findById(group._id).select('+nameTokens').lean(); + expect(stored?.nameTokens).toEqual(['platform', 'team']); + }); +}); + +describe('search query plan', () => { + const planStages = (plan: unknown): string[] => { + const stages: string[] = []; + const visit = (node: unknown) => { + if (node == null || typeof node !== 'object') { + return; + } + const record = node as Record; + if (typeof record.stage === 'string') { + stages.push(record.stage); + } + for (const value of Object.values(record)) { + visit(value); + } + }; + visit(plan); + return stages; + }; + + beforeEach(async () => { + await User.createIndexes(); + await User.create([ + user('John Smith', 'john.smith@acme.com', 'jsmith'), + user('Bob Johnson', 'bob@initech.net', 'bobby'), + ]); + }); + + it.each(['john', 'john smi', 'john.smith@acme'])( + 'uses index scans, never a collection scan, for "%s"', + async (query) => { + const filter = buildUserSearchFilter(query)!; + const explain = (await User.find(filter).limit(10).explain('queryPlanner')) as unknown as { + queryPlanner: { winningPlan: unknown }; + }; + const stages = planStages(explain.queryPlanner.winningPlan); + expect(stages).toContain('IXSCAN'); + expect(stages).not.toContain('COLLSCAN'); + }, + ); + + it('uses index scans with the admin search sort by name', async () => { + const filter = buildUserSearchFilter('jo')!; + const explain = (await User.find(filter) + .sort({ name: 1 }) + .limit(20) + .explain('queryPlanner')) as unknown as { queryPlanner: { winningPlan: unknown } }; + const stages = planStages(explain.queryPlanner.winningPlan); + expect(stages).toContain('IXSCAN'); + expect(stages).not.toContain('COLLSCAN'); + }); + + it('uses index scans under a tenant scope', async () => { + const filter = buildUserSearchFilter('john')!; + const explain = (await tenantStorage.run({ tenantId: 'tenant-a' }, () => + User.find(filter).limit(10).explain('queryPlanner').exec(), + )) as unknown as { queryPlanner: { winningPlan: unknown } }; + const stages = planStages(explain.queryPlanner.winningPlan); + expect(stages).toContain('IXSCAN'); + expect(stages).not.toContain('COLLSCAN'); + }); + + it('seeks only the scoped tenant keys when every tenant shares the prefix', async () => { + /** The scoped tenant is large with two matches; every other tenant is all matches, + * so neither a tenant-only nor a token-first seek can stay small. */ + const doc = (name: string, i: number, tenant: string) => ({ + name: `${name} ${i}`, + email: `${name.toLowerCase()}${i}@${tenant}.io`, + tenantId: tenant, + nameTokens: [name.toLowerCase(), String(i)], + emailTokens: [ + `${name.toLowerCase()}${i}@${tenant}.io`, + `${name.toLowerCase()}${i}`, + tenant, + 'io', + ], + usernameTokens: [], + }); + await User.collection.insertMany([ + ...Array.from({ length: 300 }, (_, i) => doc(i < 2 ? 'Johnny' : 'Anna', i, 'tenant-3')), + ...Array.from({ length: 300 }, (_, i) => doc('Johnny', i, `tenant-${10 + (i % 10)}`)), + ]); + const filter = buildUserSearchFilter('johnny')!; + const explain = (await tenantStorage.run({ tenantId: 'tenant-3' }, () => + User.find(filter).explain('executionStats').exec(), + )) as unknown as { executionStats: { nReturned: number; totalKeysExamined: number } }; + expect(explain.executionStats.nReturned).toBe(2); + expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(10); + }); +}); diff --git a/packages/data-schemas/src/models/plugins/searchTokens.ts b/packages/data-schemas/src/models/plugins/searchTokens.ts new file mode 100644 index 0000000000..67b935b57b --- /dev/null +++ b/packages/data-schemas/src/models/plugins/searchTokens.ts @@ -0,0 +1,71 @@ +import type { Schema, Query } from 'mongoose'; +import type { SearchTokenField } from '~/utils/search'; +import { withSearchTokens, searchTokenIndexes, computeSearchTokens } from '~/utils/search'; + +/** + * Adds the derived search-token arrays of `fields` to `schema`, indexes each + * with `tenantId` in both key orders (see `searchTokenIndexes`), and keeps them in sync on `save`, `insertMany` and the + * update queries. `bulkWrite` runs no middleware: callers that change a source + * field through it must apply `withSearchTokens` themselves. + * + * Token fields are `select: false` and stripped from `toJSON`/`toObject` + * output (a freshly saved document still holds them); they exist for the + * index, not for readers. + */ +export function applySearchTokens(schema: Schema, fields: readonly SearchTokenField[]): void { + for (const field of fields) { + schema.add({ [field.tokens]: { type: [String], default: undefined, select: false } }); + for (const index of searchTokenIndexes(field)) { + schema.index(index); + } + } + + for (const option of ['toJSON', 'toObject'] as const) { + schema.set(option, { + ...schema.get(option), + transform(_doc, ret: Record) { + for (const field of fields) { + delete ret[field.tokens]; + } + return ret; + }, + }); + } + + schema.pre('save', function () { + for (const field of fields) { + if (this.isNew || this.isModified(field.source)) { + this.set(field.tokens, computeSearchTokens(field.kind, this.get(field.source))); + } + } + }); + + schema.pre( + ['findOneAndUpdate', 'updateOne', 'updateMany'], + { document: false, query: true }, + function (this: Query) { + const update = this.getUpdate(); + if (!update) { + return; + } + const next = withSearchTokens(fields, update, { + upsert: this.getOptions().upsert === true, + filter: this.getFilter(), + }); + if (next !== update) { + this.setUpdate(next); + } + }, + ); + + schema.pre('insertMany', function (next, docs: unknown) { + if (Array.isArray(docs)) { + for (const doc of docs as Array>) { + for (const field of fields) { + doc[field.tokens] = computeSearchTokens(field.kind, doc[field.source]); + } + } + } + next(); + }); +} diff --git a/packages/data-schemas/src/schema/group.ts b/packages/data-schemas/src/schema/group.ts index bb19e4aa81..17c5e1bd06 100644 --- a/packages/data-schemas/src/schema/group.ts +++ b/packages/data-schemas/src/schema/group.ts @@ -1,5 +1,7 @@ import { Schema } from 'mongoose'; import type { IGroup } from '~/types'; +import { applySearchTokens } from '~/models/plugins/searchTokens'; +import { GROUP_SEARCH_TOKEN_FIELDS } from '~/utils/search'; const groupSchema: Schema = new Schema( { @@ -57,5 +59,6 @@ groupSchema.index( }, ); groupSchema.index({ memberIds: 1, tenantId: 1 }); +applySearchTokens(groupSchema, GROUP_SEARCH_TOKEN_FIELDS); export default groupSchema; diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.ts index 87ba658ce0..0b7329d322 100644 --- a/packages/data-schemas/src/schema/user.ts +++ b/packages/data-schemas/src/schema/user.ts @@ -1,5 +1,7 @@ import { Schema } from 'mongoose'; import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider'; +import { applySearchTokens } from '~/models/plugins/searchTokens'; +import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search'; import { IUser } from '~/types'; // Session sub-schema @@ -244,6 +246,7 @@ userSchema.index({ idOnTheSource: 1, openidIssuer: 1, tenantId: 1 }); them. The rare cleanup that pulls a deleted agent from every user names the tenant too; the one that does not (`$in` over a batch of ids) is a maintenance write, not a page. */ userSchema.index({ tenantId: 1, 'favorites.agentId': 1 }); +applySearchTokens(userSchema, USER_SEARCH_TOKEN_FIELDS); const oAuthIdFields = [ 'googleId', diff --git a/packages/data-schemas/src/types/group.ts b/packages/data-schemas/src/types/group.ts index 15cb6f288e..f935b9837f 100644 --- a/packages/data-schemas/src/types/group.ts +++ b/packages/data-schemas/src/types/group.ts @@ -15,6 +15,9 @@ export interface IGroup extends Document { createdAt?: Date; updatedAt?: Date; tenantId?: string; + /** Derived search tokens (see `~/utils/search`); `select: false`. */ + nameTokens?: string[]; + emailTokens?: string[]; } export interface CreateGroupRequest { diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.ts index e2b3cd861b..9b0e8837b8 100644 --- a/packages/data-schemas/src/types/user.ts +++ b/packages/data-schemas/src/types/user.ts @@ -89,6 +89,10 @@ export interface IUser extends Document { /** Field for external source identification (for consistency with TPrincipal schema) */ idOnTheSource?: string; tenantId?: string; + /** Derived search tokens (see `~/utils/search`); `select: false`. */ + nameTokens?: string[]; + emailTokens?: string[]; + usernameTokens?: string[]; federatedTokens?: OIDCTokens; openidTokens?: OIDCTokens; } diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts index f5a4c6fd48..828eb80429 100644 --- a/packages/data-schemas/src/utils/index.ts +++ b/packages/data-schemas/src/utils/index.ts @@ -1,5 +1,6 @@ export * from './principal'; export * from './string'; +export * from './search'; export * from './tempChatRetention'; export * from './retention'; export { tenantSafeBulkWrite } from './tenantBulkWrite'; diff --git a/packages/data-schemas/src/utils/search.spec.ts b/packages/data-schemas/src/utils/search.spec.ts new file mode 100644 index 0000000000..db38aebcb6 --- /dev/null +++ b/packages/data-schemas/src/utils/search.spec.ts @@ -0,0 +1,215 @@ +import { + withSearchTokens, + computeSearchTokens, + normalizeSearchText, + buildUserSearchFilter, + MAX_SEARCH_TOKENS, + USER_SEARCH_TOKEN_FIELDS, + MAX_SEARCH_TOKEN_LENGTH, + MAX_SEARCH_QUERY_TOKENS, +} from './search'; + +describe('normalizeSearchText', () => { + it('lowercases and folds accents and compatibility forms', () => { + expect(normalizeSearchText('José ÅNGSTRÖM')).toBe('jose angstrom'); + expect(normalizeSearchText('Full')).toBe('full'); + expect(normalizeSearchText('İstanbul')).toBe('istanbul'); + }); +}); + +describe('computeSearchTokens', () => { + it('splits names on whitespace and punctuation', () => { + expect(computeSearchTokens('words', " Mary-Jane O'Brien, Jr. ")).toEqual([ + 'mary', + 'jane', + 'o', + 'brien', + 'jr', + ]); + }); + + it('keeps non-Latin scripts as words', () => { + expect(computeSearchTokens('words', '山田 太郎')).toEqual(['山田', '太郎']); + expect(computeSearchTokens('words', 'Ζωή Κ.')).toEqual(['ζωη', 'κ']); + }); + + it('indexes the full email, local-part words, and domain labels', () => { + expect(computeSearchTokens('email', 'John.Smith+work@Mail.Example-Co.com')).toEqual([ + 'john.smith+work@mail.example-co.com', + 'john', + 'smith', + 'work', + 'mail', + 'example', + 'co', + 'com', + 'example-co', + ]); + }); + + it('indexes the full username and its parts', () => { + expect(computeSearchTokens('handle', 'jane_doe.99')).toEqual([ + 'jane_doe.99', + 'jane', + 'doe', + '99', + ]); + }); + + it('dedupes and drops empties', () => { + expect(computeSearchTokens('words', 'Ann ann ANN --')).toEqual(['ann']); + expect(computeSearchTokens('words', ' ')).toEqual([]); + expect(computeSearchTokens('words', '')).toEqual([]); + }); + + it('returns no tokens for non-strings', () => { + expect(computeSearchTokens('words', undefined)).toEqual([]); + expect(computeSearchTokens('email', null)).toEqual([]); + expect(computeSearchTokens('handle', 42)).toEqual([]); + }); + + it('caps token length and token count', () => { + const long = 'a'.repeat(MAX_SEARCH_TOKEN_LENGTH + 10); + expect(computeSearchTokens('words', long)).toEqual(['a'.repeat(MAX_SEARCH_TOKEN_LENGTH)]); + + const many = Array.from({ length: MAX_SEARCH_TOKENS + 10 }, (_, i) => `w${i}`).join(' '); + const tokens = computeSearchTokens('words', many); + expect(tokens).toHaveLength(MAX_SEARCH_TOKENS); + expect(tokens[0]).toBe('w0'); + }); + + it('does not split a surrogate pair when truncating', () => { + const [token] = computeSearchTokens('words', '\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH + 5)); + expect(Array.from(token)).toHaveLength(MAX_SEARCH_TOKEN_LENGTH); + expect(token).toBe('\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH)); + }); +}); + +describe('withSearchTokens', () => { + it('derives tokens for each source field in $set and leaves the input untouched', () => { + const update = { $set: { name: 'Ana Lima', role: 'USER' } }; + const next = withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update); + expect(next).toEqual({ $set: { name: 'Ana Lima', role: 'USER', nameTokens: ['ana', 'lima'] } }); + expect(update).toEqual({ $set: { name: 'Ana Lima', role: 'USER' } }); + }); + + it('handles top-level, $setOnInsert and $unset writes', () => { + expect( + withSearchTokens(USER_SEARCH_TOKEN_FIELDS, { + username: 'al', + $setOnInsert: { email: 'a@b.io' }, + $unset: { name: '' }, + }), + ).toEqual({ + username: 'al', + usernameTokens: ['al'], + $setOnInsert: { email: 'a@b.io', emailTokens: ['a@b.io', 'a', 'b', 'io'] }, + $unset: { name: '' }, + $set: { nameTokens: [] }, + }); + }); + + it('ignores undefined assignments, which Mongoose drops from the update', () => { + const update = { $set: { name: undefined, role: 'ADMIN' }, username: undefined }; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update); + }); + + it('ignores an undefined $unset, which Mongoose also drops', () => { + const update = { $unset: { name: undefined } }; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update); + }); + + it('initializes every unwritten token field on upsert inserts', () => { + expect( + withSearchTokens(USER_SEARCH_TOKEN_FIELDS, { $set: { name: 'Al' } }, { upsert: true }), + ).toEqual({ + $set: { name: 'Al', nameTokens: ['al'] }, + $setOnInsert: { emailTokens: [], usernameTokens: [] }, + }); + }); + + it('takes upsert insert tokens from equality conditions the insert copies', () => { + expect( + withSearchTokens( + USER_SEARCH_TOKEN_FIELDS, + { $set: { name: 'Al' } }, + { upsert: true, filter: { email: 'al@x.io', username: { $eq: 'al' } } }, + ), + ).toEqual({ + $set: { name: 'Al', nameTokens: ['al'] }, + $setOnInsert: { emailTokens: ['al@x.io', 'al', 'x', 'io'], usernameTokens: ['al'] }, + }); + expect( + withSearchTokens( + USER_SEARCH_TOKEN_FIELDS, + { $set: { name: 'Al' } }, + { upsert: true, filter: { email: { $in: ['a@x.io', 'b@x.io'] } } }, + ), + ).toEqual({ + $set: { name: 'Al', nameTokens: ['al'] }, + $setOnInsert: { usernameTokens: [] }, + }); + }); + + it('returns the same reference when no source field is written', () => { + const update = { $set: { role: 'ADMIN' } }; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update); + const pipeline = [{ $set: { name: 'x' } }]; + expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, pipeline)).toBe(pipeline); + }); +}); + +describe('buildUserSearchFilter', () => { + it('returns null when the query has nothing to match', () => { + expect(buildUserSearchFilter('')).toBeNull(); + expect(buildUserSearchFilter(' ')).toBeNull(); + }); + + it('matches a single token by anchored, case-sensitive prefix on every token field', () => { + const filter = buildUserSearchFilter(' José ') as { $or: Array> }; + const prefixes = filter.$or.filter((branch) => Object.keys(branch).length === 1); + expect(prefixes).toEqual([ + { nameTokens: /^jose/ }, + { emailTokens: /^jose/ }, + { usernameTokens: /^jose/ }, + ]); + expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /José/i }); + }); + + it('anchors the legacy fallback for callers whose old search was a prefix match', () => { + const filter = buildUserSearchFilter('mith', { legacyPrefix: true })!; + expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /^mith/i }); + expect(filter.$or).not.toContainEqual({ nameTokens: { $exists: false }, name: /mith/i }); + }); + + it('requires every query token and also tries the whole query on full-value fields', () => { + const filter = buildUserSearchFilter('john smi') as { $and: Array<{ $or: unknown[] }> }; + expect(filter.$and).toHaveLength(2); + expect(filter.$and[0].$or).toContainEqual({ nameTokens: /^john/ }); + expect(filter.$and[1].$or).toContainEqual({ nameTokens: /^smi/ }); + expect(filter.$and[1].$or).toContainEqual({ emailTokens: /^john smi/ }); + expect(filter.$and[1].$or).not.toContainEqual({ nameTokens: /^john smi/ }); + }); + + it('escapes regex metacharacters and requires every query token up to the cap', () => { + const filter = buildUserSearchFilter('a.b') as { $and: Array<{ $or: unknown[] }> }; + expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^a\.b/ }); + + const words = (n: number) => Array.from({ length: n }, (_, i) => `t${i}`).join(' '); + const all = buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS)) as { + $and: Array<{ $or: unknown[] }>; + }; + expect(all.$and).toHaveLength(MAX_SEARCH_QUERY_TOKENS); + expect(all.$and[MAX_SEARCH_QUERY_TOKENS - 1].$or).toContainEqual({ + nameTokens: new RegExp(`^t${MAX_SEARCH_QUERY_TOKENS - 1}`), + }); + /** Over the cap, nothing matches: extra words are never silently dropped. */ + expect(buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS + 1))).toBeNull(); + }); + + it('searches punctuation-only queries on full values only', () => { + const filter = buildUserSearchFilter('.*') as { $or: unknown[] }; + expect(filter.$or).toContainEqual({ emailTokens: /^\.\*/ }); + expect(filter.$or).not.toContainEqual({ nameTokens: expect.anything() }); + }); +}); diff --git a/packages/data-schemas/src/utils/search.ts b/packages/data-schemas/src/utils/search.ts new file mode 100644 index 0000000000..63a232561d --- /dev/null +++ b/packages/data-schemas/src/utils/search.ts @@ -0,0 +1,328 @@ +import { escapeRegExp } from './string'; + +/** + * Word-prefix search over derived token arrays. + * + * Each searchable source field (`name`, `email`, `username`) gets a lowercase, + * accent-folded token array stored beside it and indexed with `tenantId`. A query + * is tokenized the same way, and each query token must prefix-match some token, + * so "john smi" finds "John Smith" through an anchored, case-sensitive regex + * that the multikey index can bound. Matches inside a word ("ohn") are not + * found, by design: that is the price of an index-bound search. + * + * The token rules are a stored format: changing them requires re-running the + * backfill, so the caps are constants rather than configuration. + */ + +/** How a source value is split into tokens. */ +export type SearchTokenKind = 'words' | 'email' | 'handle'; + +export interface SearchTokenField { + /** The stored field the tokens derive from. */ + readonly source: string; + /** The derived, indexed token array. */ + readonly tokens: string; + readonly kind: SearchTokenKind; +} + +export const USER_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [ + { source: 'name', tokens: 'nameTokens', kind: 'words' }, + { source: 'email', tokens: 'emailTokens', kind: 'email' }, + { source: 'username', tokens: 'usernameTokens', kind: 'handle' }, +]; + +export const GROUP_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [ + { source: 'name', tokens: 'nameTokens', kind: 'words' }, + { source: 'email', tokens: 'emailTokens', kind: 'email' }, +]; + +/** Longest stored or queried word token; longer words are truncated to this prefix. */ +export const MAX_SEARCH_TOKEN_LENGTH = 64; +/** Longest whole-value token (a full email address or username). */ +export const MAX_SEARCH_VALUE_LENGTH = 256; +/** Most tokens stored per field. */ +export const MAX_SEARCH_TOKENS = 32; +/** + * Most distinct query words. Every word must match, so a longer query matches + * nothing rather than ignoring its extra words; the cap bounds the filter a + * single request can build. + */ +export const MAX_SEARCH_QUERY_TOKENS = 16; + +const NON_WORD = /[^\p{L}\p{N}]+/u; +const COMBINING_MARKS = /\p{M}+/gu; + +function truncate(value: string, max: number): string { + if (value.length <= max) { + return value; + } + return Array.from(value).slice(0, max).join(''); +} + +/** Lowercases and folds accents, so "José" and "jose" compare equal. */ +export function normalizeSearchText(value: string): string { + return value.normalize('NFKD').replace(COMBINING_MARKS, '').toLowerCase(); +} + +function words(normalized: string): string[] { + return normalized + .split(NON_WORD) + .filter(Boolean) + .map((word) => truncate(word, MAX_SEARCH_TOKEN_LENGTH)); +} + +/** + * A value reduced to the words the filter matches, joined by single spaces, so + * relevance scoring sees "Mary-Jane" and "mary jane" as the same text. + */ +export function normalizeSearchPhrase(value: string): string { + return words(normalizeSearchText(value)).join(' '); +} + +/** + * Relevance of a matched document for `query`, over the values the filter + * searched. Like the filter, it ignores word order: 100 when a value has the + * query's words in any order, 80 when a value starts with the query, 50 when a + * value contains it or every query word prefixes some word (in one value or + * across them), 10 otherwise. + */ +export function scoreSearchMatch( + values: ReadonlyArray, + query: string, +): number { + const phrase = normalizeSearchPhrase(query); + if (!phrase) { + return 10; + } + const queryWords = phrase.split(' '); + const sortedQuery = [...queryWords].sort().join(' '); + const valueWords: string[] = []; + let score = 10; + for (const value of values) { + if (!value) { + continue; + } + const field = normalizeSearchPhrase(value); + const words = field.split(' '); + if (field === phrase || [...words].sort().join(' ') === sortedQuery) { + return 100; + } + if (field.startsWith(phrase)) { + score = Math.max(score, 80); + } else if (field.includes(phrase)) { + score = Math.max(score, 50); + } + valueWords.push(...words); + } + const everyWord = queryWords.every((queryWord) => + valueWords.some((word) => word.startsWith(queryWord)), + ); + return everyWord ? Math.max(score, 50) : score; +} + +/** Derives the stored token array for one field value. Non-strings yield no tokens. */ +export function computeSearchTokens(kind: SearchTokenKind, value: unknown): string[] { + if (typeof value !== 'string') { + return []; + } + const normalized = normalizeSearchText(value.trim()); + if (!normalized) { + return []; + } + const tokens: string[] = []; + if (kind !== 'words') { + tokens.push(truncate(normalized, MAX_SEARCH_VALUE_LENGTH)); + } + tokens.push(...words(normalized)); + if (kind === 'email') { + const domain = normalized.slice(normalized.lastIndexOf('@') + 1); + tokens.push(...domain.split('.').filter(Boolean)); + } + return [...new Set(tokens)].slice(0, MAX_SEARCH_TOKENS); +} + +/** Every token field of `fields`, computed from a document's source values. */ +export function computeSearchTokenSet( + fields: readonly SearchTokenField[], + doc: Record, +): Record { + const result: Record = {}; + for (const field of fields) { + result[field.tokens] = computeSearchTokens(field.kind, doc[field.source]); + } + return result; +} + +type UpdateDoc = Record; + +/** + * The indexes behind each token field. Tenant first serves tenant-scoped + * searches with an equality seek before the prefix range; token first serves + * unscoped ones (single-tenant deployments run without a tenant context), which + * could not use the tenant-first index at all. + */ +export function searchTokenIndexes(field: SearchTokenField): Array> { + return [ + { tenantId: 1, [field.tokens]: 1 }, + { [field.tokens]: 1, tenantId: 1 }, + ]; +} + +/** Mongoose drops `undefined` assignments from an update, so they write nothing. */ +function writes(doc: UpdateDoc | undefined, key: string): doc is UpdateDoc { + return doc != null && key in doc && doc[key] !== undefined; +} + +const SKIP = Symbol('skip'); + +/** + * The value an upsert insert takes for `key` from the query filter: a plain or + * `$eq` equality is copied into the new document, no condition leaves it unset. + * Any other condition is ambiguous, so the token field is left for the backfill. + */ +function insertedValue(filter: UpdateDoc | undefined, key: string): unknown { + const condition = filter?.[key]; + if (condition === undefined) { + return undefined; + } + if (typeof condition === 'string') { + return condition; + } + const eq = operator({ condition }, 'condition'); + return eq && Object.keys(eq).length === 1 && typeof eq.$eq === 'string' ? eq.$eq : SKIP; +} + +function operator(update: UpdateDoc, key: string): UpdateDoc | undefined { + const value = update[key]; + return value != null && typeof value === 'object' && !Array.isArray(value) + ? (value as UpdateDoc) + : undefined; +} + +/** + * Returns `update` with the token fields recomputed for every source field it + * writes, through `$set`, `$setOnInsert`, `$unset` or a top-level replacement + * value. Each token field depends on one source field, so a partial update + * stays correct. On an upsert, a token field whose source the update does not + * write is initialized on insert from the source's equality condition in + * `filter` (MongoDB copies it into the new document), or to `[]` when the + * filter has none, so new documents never look un-migrated. Copy-on-write: the same reference comes back when nothing + * changes. Pipeline updates are returned untouched. + * + * Mongoose runs no middleware for `bulkWrite`; a bulk write that changes a + * source field must pass its update through this function. + */ +export function withSearchTokens( + fields: readonly SearchTokenField[], + update: T, + options: { upsert?: boolean; filter?: UpdateDoc } = {}, +): T { + if (update == null || typeof update !== 'object' || Array.isArray(update)) { + return update; + } + const source = update as UpdateDoc; + let next: UpdateDoc | undefined; + const write = (op: string | null, key: string, tokens: string[]) => { + next ??= { ...source }; + if (op == null) { + next[key] = tokens; + return; + } + next[op] = { ...(operator(next, op) ?? {}), [key]: tokens }; + }; + + for (const field of fields) { + const set = operator(source, '$set'); + const setOnInsert = operator(source, '$setOnInsert'); + const unset = operator(source, '$unset'); + let written = true; + if (writes(set, field.source)) { + write('$set', field.tokens, computeSearchTokens(field.kind, set[field.source])); + } else if (writes(source, field.source)) { + write(null, field.tokens, computeSearchTokens(field.kind, source[field.source])); + } else if (writes(unset, field.source)) { + write('$set', field.tokens, []); + } else { + written = false; + } + if (writes(setOnInsert, field.source)) { + write( + '$setOnInsert', + field.tokens, + computeSearchTokens(field.kind, setOnInsert[field.source]), + ); + } else if (options.upsert && !written) { + const condition = insertedValue(options.filter, field.source); + if (condition !== SKIP) { + write('$setOnInsert', field.tokens, computeSearchTokens(field.kind, condition)); + } + } + } + return (next ?? source) as T; +} + +export interface SearchFilterOptions { + /** Anchor the legacy fallback at the start of the field, for callers whose old search was a prefix match. */ + legacyPrefix?: boolean; +} + +/** + * Builds the filter for a word-prefix search, or `null` when the query has + * nothing to match. + * + * Every query token must prefix-match a token of some field (fields may + * differ per token, so "john gmail" finds John at gmail.com). Fields that + * store their whole value (email, username) also match the whole query as a + * prefix, so a pasted "john.smith@exa" works. Documents written before the + * token fields existed fall back to the caller's old case-insensitive regex + * on the source field (unanchored, or anchored with `legacyPrefix`); the + * fallback branches seek the token index on `$exists: false` and match + * nothing once the backfill has run. + */ +export function buildSearchTokenFilter( + fields: readonly SearchTokenField[], + query: string, + options: SearchFilterOptions = {}, +): Record | null { + const trimmed = query.trim(); + const normalized = normalizeSearchText(trimmed); + const queryTokens = [...new Set(words(normalized))]; + const whole = truncate(normalized, MAX_SEARCH_VALUE_LENGTH); + const matchWhole = whole.length > 0 && !(queryTokens.length === 1 && queryTokens[0] === whole); + if ( + !trimmed || + queryTokens.length > MAX_SEARCH_QUERY_TOKENS || + (queryTokens.length === 0 && !matchWhole) + ) { + return null; + } + + const legacy = new RegExp(`${options.legacyPrefix ? '^' : ''}${escapeRegExp(trimmed)}`, 'i'); + const shared: Record[] = []; + for (const field of fields) { + if (matchWhole && field.kind !== 'words') { + shared.push({ [field.tokens]: new RegExp(`^${escapeRegExp(whole)}`) }); + } + shared.push({ [field.tokens]: { $exists: false }, [field.source]: legacy }); + } + + if (queryTokens.length === 0) { + return { $or: shared }; + } + const clauses = queryTokens.map((token) => { + const prefix = new RegExp(`^${escapeRegExp(token)}`); + return { $or: [...fields.map((field) => ({ [field.tokens]: prefix })), ...shared] }; + }); + return clauses.length === 1 ? clauses[0] : { $and: clauses }; +} + +export function buildUserSearchFilter( + query: string, + options?: SearchFilterOptions, +): Record | null { + return buildSearchTokenFilter(USER_SEARCH_TOKEN_FIELDS, query, options); +} + +export function buildGroupSearchFilter(query: string): Record | null { + return buildSearchTokenFilter(GROUP_SEARCH_TOKEN_FIELDS, query); +}