mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 15:02:02 +00:00
🔎 perf: Index-Bound Word-Prefix Search for Users and Groups (#16719)
* perf: Debounce People Picker Search * perf: Index-Bound Word-Prefix Search for Users and Groups Principal search and admin user search matched name, email and username with case-insensitive regexes, which scan the whole users collection. Users and groups now store lowercase, accent-folded token arrays derived from each searchable field, indexed with tenantId, and searches match anchored token prefixes. Schema middleware keeps the tokens in sync on save, insertMany and update queries. Documents written before the tokens existed stay findable through a fallback branch until npm run migrate:search-tokens backfills them; startup warns while any remain. * perf: Build Search Token Indexes in the Backfill Migration * fix: Reconcile Stale Search Tokens and Close Write-Path Gaps The backfill now compares every document's tokens with its source fields and repairs stale ones, so writes from an older server during a rolling deploy are recoverable by rerunning it. Undefined assignments no longer clear tokens, upsert inserts initialize every token field, every query word is required, relevance scoring folds accents like the filter does, and the startup check counts both collections in parallel and owns its error logging. * perf: Drop the Second People Picker Debounce SearchPicker already debounces query changes by 500ms; the extra 300ms delay in UnifiedPeopleSearch stacked on top of it. * fix: Match Nothing for Queries Over the Word Cap Dropping the query word cap let one request build an unbounded filter. Restore a 16 word cap, but a longer query now matches nothing instead of ignoring its extra words, so a cap never produces a false positive. * test: Cover Word-Prefix People Search and the Token Backfill End to End * fix: Tokenize Fields an Upsert Copies From Its Filter MongoDB copies equality conditions from an upsert's filter into the inserted document, so their token fields now derive from that value instead of being initialized empty. * fix: Rank Accent-Folded User Matches Like the Search Filter * test: Run the Token Backfill Without Blocking the Playwright Worker * perf: Index Search Tokens Tenant First for Scoped Searches and Ignore Undefined Unsets * test: Skip the Module-Scope Index Rebuild When the Scenario Runs the Backfill * fix: Exit the Search Token Migration When Redis Clients Stay Open * perf: Probe for Missing Search Tokens at Startup Instead of Counting Them * test: Explain the Startup Probe the Warning Actually Issues * fix: Keep Search Token Fields Out of Serialized Users and Groups * fix: Keep the Admin Search Legacy Fallback Anchored at the Field Start * fix: Score Groups Only on the Fields Search Matches * perf: Start the Search Token Probe Alongside the Permission Checks * fix: Warn at Startup When the Search Token Indexes Were Never Built * fix: Score People Search Matches With the Filter's Word Separators * perf: Check the Token Indexes Before Probing for Unmigrated Documents * test: Drive the Failed Startup Probe Past the Index Check * fix: Score People Search Matches Independently of Word Order
This commit is contained in:
parent
f757a673aa
commit
de9d46fcd4
25 changed files with 1834 additions and 123 deletions
|
|
@ -1,4 +1,5 @@
|
|||
import { Types } from 'mongoose';
|
||||
import { buildUserSearchFilter } from '@librechat/data-schemas';
|
||||
import { PrincipalType, SystemRoles } from 'librechat-data-provider';
|
||||
import type { IUser, UserDeleteResult } from '@librechat/data-schemas';
|
||||
import type { Response } from 'express';
|
||||
|
|
@ -205,20 +206,23 @@ describe('createAdminUsersHandlers', () => {
|
|||
expect(response.capped).toBe(true);
|
||||
});
|
||||
|
||||
it('searches name, email, and username with anchored prefix regex', async () => {
|
||||
it('searches the indexed word-prefix tokens of name, email, and username', async () => {
|
||||
const findUsers = jest.fn().mockResolvedValue([]);
|
||||
const deps = createDeps({ findUsers });
|
||||
const handlers = createAdminUsersHandlers(deps);
|
||||
const { req, res } = createReqRes({ query: { q: 'test' } });
|
||||
const { req, res } = createReqRes({ query: { q: 'Test' } });
|
||||
|
||||
await handlers.searchUsers(req, res);
|
||||
|
||||
const filter = findUsers.mock.calls[0][0];
|
||||
expect(filter.$or).toHaveLength(3);
|
||||
expect(filter.$or[0]).toHaveProperty('name');
|
||||
expect(filter.$or[1]).toHaveProperty('email');
|
||||
expect(filter.$or[2]).toHaveProperty('username');
|
||||
expect(filter.$or[0].name.source).toBe('^test');
|
||||
expect(filter).toEqual(buildUserSearchFilter('Test', { legacyPrefix: true }));
|
||||
expect(filter.$or).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ nameTokens: /^test/ },
|
||||
{ emailTokens: /^test/ },
|
||||
{ usernameTokens: /^test/ },
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('projects username in the field selection', async () => {
|
||||
|
|
@ -242,8 +246,25 @@ describe('createAdminUsersHandlers', () => {
|
|||
await handlers.searchUsers(req, res);
|
||||
|
||||
const filter = findUsers.mock.calls[0][0];
|
||||
expect(filter.$or[0].name).toBeInstanceOf(RegExp);
|
||||
expect(filter.$or[0].name.source).toBe('^test\\.user\\+1');
|
||||
expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^test\.user\+1/ });
|
||||
/** Documents awaiting the backfill keep the old anchored prefix match. */
|
||||
expect(filter.$and[0].$or).toContainEqual({
|
||||
nameTokens: { $exists: false },
|
||||
name: /^test\.user\+1/i,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns no users without querying when the query has nothing to match', async () => {
|
||||
const findUsers = jest.fn();
|
||||
const deps = createDeps({ findUsers });
|
||||
const handlers = createAdminUsersHandlers(deps);
|
||||
const { req, res, status, json } = createReqRes({ query: { q: '\u0301\u0301' } });
|
||||
|
||||
await handlers.searchUsers(req, res);
|
||||
|
||||
expect(findUsers).not.toHaveBeenCalled();
|
||||
expect(status).toHaveBeenCalledWith(200);
|
||||
expect(json).toHaveBeenCalledWith({ users: [], total: 0, capped: false });
|
||||
});
|
||||
|
||||
it('returns 400 when query is missing', async () => {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { Types } from 'mongoose';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { PrincipalType, SystemRoles } from 'librechat-data-provider';
|
||||
import { logger, isValidObjectIdString } from '@librechat/data-schemas';
|
||||
import { logger, isValidObjectIdString, buildUserSearchFilter } from '@librechat/data-schemas';
|
||||
import type {
|
||||
IUser,
|
||||
IConfig,
|
||||
|
|
@ -197,14 +197,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
|
|||
}
|
||||
|
||||
const searchLimit = Math.min(Math.max(1, parseInt(limitStr, 10) || 20), 50);
|
||||
const escaped = trimmed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const regex = new RegExp(`^${escaped}`, 'i');
|
||||
|
||||
const users = await findUsers(
|
||||
{ $or: [{ name: regex }, { email: regex }, { username: regex }] },
|
||||
'_id name email username avatar',
|
||||
{ limit: searchLimit, sort: { name: 1 } },
|
||||
);
|
||||
const filter = buildUserSearchFilter(trimmed, { legacyPrefix: true });
|
||||
const users = filter
|
||||
? await findUsers(filter, '_id name email username avatar', {
|
||||
limit: searchLimit,
|
||||
sort: { name: 1 },
|
||||
})
|
||||
: [];
|
||||
|
||||
const results: AdminUserSearchResult[] = users.map((u) => ({
|
||||
id: u._id?.toString() ?? '',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue