diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.js index 1edae18156..53d73125df 100644 --- a/api/cache/getLogStores.js +++ b/api/cache/getLogStores.js @@ -66,7 +66,9 @@ const namespaces = { CacheKeys.OPENID_EXCHANGED_TOKENS, Time.TEN_MINUTES, ), - [CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC), + [CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC, undefined, undefined, { + throwOnErrors: true, + }), [CacheKeys.ADMIN_OAUTH_EXCHANGE]: standardCache( CacheKeys.ADMIN_OAUTH_EXCHANGE, Time.THIRTY_SECONDS, diff --git a/api/server/controllers/AuthController.js b/api/server/controllers/AuthController.js index 107f0dbdff..f5e3a230f0 100644 --- a/api/server/controllers/AuthController.js +++ b/api/server/controllers/AuthController.js @@ -24,6 +24,7 @@ const { const { deleteAllUserSessions, deletePasskeysByUser, + awaitAuthUserDocEviction, getUserById, findSession, updateUser, @@ -326,6 +327,7 @@ const resetPasswordController = createResetPasswordController({ resetPassword, deleteAllUserSessions, deletePasskeysByUser, + awaitAuthUserDocEviction, }); const refreshController = async (req, res) => { diff --git a/api/server/controllers/AuthController.spec.js b/api/server/controllers/AuthController.spec.js index 50090636ea..6a1866c1f9 100644 --- a/api/server/controllers/AuthController.spec.js +++ b/api/server/controllers/AuthController.spec.js @@ -42,6 +42,7 @@ jest.mock('~/models', () => ({ deleteSession: jest.fn(), deleteAllUserSessions: jest.fn(), deletePasskeysByUser: jest.fn(), + awaitAuthUserDocEviction: jest.fn(), getUserById: jest.fn(), findSession: jest.fn(), updateUser: jest.fn(), @@ -147,6 +148,7 @@ const { deleteTokens, deleteAllUserSessions, deletePasskeysByUser, + awaitAuthUserDocEviction, } = require('~/models'); const { createRefreshTokenBridgeFlightKey, @@ -2483,6 +2485,7 @@ describe('resetPasswordController', () => { resetPassword.mockReset().mockResolvedValue(serviceResult); deleteAllUserSessions.mockReset().mockResolvedValue(undefined); deletePasskeysByUser.mockReset().mockResolvedValue(undefined); + awaitAuthUserDocEviction.mockReset().mockResolvedValue(undefined); req = { body: { userId: 'user-123', @@ -2511,6 +2514,31 @@ describe('resetPasswordController', () => { expect(res.json).toHaveBeenCalledWith(serviceResult); }); + it('confirms the reset only after revoking and settling the auth cache', async () => { + let finishEviction; + awaitAuthUserDocEviction.mockReturnValue( + new Promise((resolve) => { + finishEviction = resolve; + }), + ); + + const reset = resetPasswordController(req, res); + await new Promise((resolve) => setImmediate(resolve)); + const responseCalls = res.status.mock.calls.length; + finishEviction(); + await reset; + + expect(awaitAuthUserDocEviction).toHaveBeenCalledWith('user-123'); + expect(deleteAllUserSessions.mock.invocationCallOrder[0]).toBeLessThan( + awaitAuthUserDocEviction.mock.invocationCallOrder[0], + ); + expect(deletePasskeysByUser.mock.invocationCallOrder[0]).toBeLessThan( + awaitAuthUserDocEviction.mock.invocationCallOrder[0], + ); + expect(responseCalls).toBe(0); + expect(res.status).toHaveBeenCalledWith(200); + }); + it('does not revoke sessions or passkeys when reset fails', async () => { const resetError = new Error('Invalid token'); resetPassword.mockResolvedValue(resetError); @@ -2519,6 +2547,7 @@ describe('resetPasswordController', () => { expect(deleteAllUserSessions).not.toHaveBeenCalled(); expect(deletePasskeysByUser).not.toHaveBeenCalled(); + expect(awaitAuthUserDocEviction).not.toHaveBeenCalled(); expect(res.status).toHaveBeenCalledWith(400); expect(res.json).toHaveBeenCalledWith(resetError); }); diff --git a/config/reset-password.js b/config/reset-password.js index 1d239f58b1..3144ade839 100644 --- a/config/reset-password.js +++ b/config/reset-password.js @@ -85,9 +85,15 @@ const resetPassword = async () => { * A passkey signs in on its own, so leaving one in place would keep an attacker * logged in after an administrator believes the account has been recovered. */ - const { deletedCount } = await Passkey.deleteMany({ user: user._id }); - /** A refresh session outlives the stamp: refreshing mints a token issued after it. */ - await Session.deleteMany({ user: user._id }); + let deletedCount; + try { + ({ deletedCount } = await Passkey.deleteMany({ user: user._id })); + /** A refresh session outlives the stamp: refreshing mints a token issued after it. */ + await Session.deleteMany({ user: user._id }); + } finally { + /** The password is already changed, so the cache settles even when a revocation fails. */ + await methods.awaitAuthUserDocEviction(user._id.toString()); + } console.log('Password successfully reset!'); if (deletedCount > 0) { diff --git a/e2e/specs/mock/scenarios/reset-password.spec.ts b/e2e/specs/mock/scenarios/reset-password.spec.ts index fc14e00759..a25739132a 100644 --- a/e2e/specs/mock/scenarios/reset-password.spec.ts +++ b/e2e/specs/mock/scenarios/reset-password.spec.ts @@ -4,7 +4,11 @@ import os from 'os'; import { Keyv } from 'keyv'; import KeyvRedis from '@keyv/redis'; import { expect, test } from '@playwright/test'; -import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider'; +import { + CacheKeys, + AUTH_USER_DOC_BY_ID_PREFIX, + AUTH_USER_DOC_CACHE_TTL_MS, +} from 'librechat-data-provider'; import { spawn } from 'node:child_process'; import { randomInt, randomUUID } from 'node:crypto'; import { withMongo, seedPasskey, deleteUserByEmail } from '../db'; @@ -40,7 +44,13 @@ function runResetCli( email: string, cwd: string, env: NodeJS.ProcessEnv, -): Promise<{ code: number | null; stdout: string; stderr: string }> { +): Promise<{ + code: number | null; + stdout: string; + stderr: string; + /** Milliseconds between confirming the new password and the CLI reporting success. */ + reportDelayMs?: number; +}> { const child = spawn(process.execPath, [path.join(REPO_ROOT, 'config/reset-password.js')], { cwd, env: { @@ -59,14 +69,20 @@ function runResetCli( ['Confirm new password: ', NEW_PASSWORD], ]; let stdout = ''; + let confirmedAt: number | undefined; + let reportedAt: number | undefined; child.stdout?.on('data', (chunk: Buffer) => { stdout += chunk.toString(); answers.forEach((answer, index) => { if (answer !== null && stdout.includes(answer[0])) { child.stdin?.write(`${answer[1]}\n`); answers[index] = null; + confirmedAt = Date.now(); } }); + if (reportedAt === undefined && stdout.includes('Password successfully reset!')) { + reportedAt = Date.now(); + } }); let stderr = ''; child.stderr?.on('data', (chunk: Buffer) => (stderr += chunk.toString())); @@ -82,7 +98,11 @@ function runResetCli( child.on('close', (code) => { clearTimeout(timer); child.stdin?.end(); - resolve({ code, stdout, stderr }); + const reportDelayMs = + confirmedAt !== undefined && reportedAt !== undefined + ? reportedAt - confirmedAt + : undefined; + resolve({ code, stdout, stderr, reportDelayMs }); }); }); } @@ -230,4 +250,85 @@ test.describe('CLI password reset', () => { } }); } + + test('reports the reset only after unevictable cached credentials expire @scenario:cli-reset-outlasts-failed-cache-eviction', async ({ + playwright, + baseURL, + }) => { + test.setTimeout(90_000); + const request = await playwright.request.newContext({ + baseURL, + extraHTTPHeaders: { + 'X-Forwarded-For': `127.${randomInt(1, 255)}.${randomInt(1, 255)}.${randomInt(1, 255)}`, + }, + }); + const email = `cli-evict-${randomUUID().slice(0, 8)}@example.com`; + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-reset-')); + const redisUri = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379'; + const prefix = `cli-evict-${randomUUID()}`; + const adapter = new KeyvRedis(redisUri, { throwOnErrors: true, connectionTimeout: 5000 }); + const cache = new Keyv(adapter, { namespace: CacheKeys.AUTH_USER_DOC }); + adapter.namespace = prefix; + adapter.keyPrefixSeparator = '::'; + const keys: string[] = []; + try { + const register = await request.post('/api/auth/register', { + data: { email, name: 'CLI Evict', password: PASSWORD, confirm_password: PASSWORD }, + }); + expect(register.ok()).toBeTruthy(); + const login = await request.post('/api/auth/login', { data: { email, password: PASSWORD } }); + expect(login.ok()).toBeTruthy(); + const { token, user } = (await login.json()) as { token: string; user: { _id: string } }; + await seedPasskey(email, randomUUID(), 'Eviction regression'); + + const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${user._id}`; + const controlKey = 'auth-user-doc:another-user'; + keys.push(indexKey, controlKey); + await cache.set(controlKey, { credentialsChangedAt: null }, 120_000); + /** A hash where the index string belongs makes the CLI's GET fail with WRONGTYPE. */ + const client = await adapter.getClient(); + await client.hSet(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 'unreadable', '1'); + await client.pExpire(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 120_000); + + const env = { ...process.env }; + Object.assign(env, { + USE_REDIS: 'true', + USE_REDIS_CLUSTER: 'false', + REDIS_URI: redisUri, + REDIS_KEY_PREFIX: prefix, + REDIS_KEY_PREFIX_VAR: '', + AUTH_USER_CACHE_MODE: 'on', + FORCED_IN_MEMORY_CACHE_NAMESPACES: '', + }); + + const cli = await runResetCli(email, cwd, env); + expect(cli.code, cli.stderr.slice(-400)).toBe(0); + expect(cli.stdout).toContain('Password successfully reset!'); + expect(cli.reportDelayMs).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS); + expect(await cache.get(controlKey)).toEqual({ credentialsChangedAt: null }); + await withMongo(async (db) => { + const resetUser = await db.collection('users').findOne({ email }); + expect(resetUser?.credentialsChangedAt).toBeTruthy(); + expect(await db.collection('passkeys').countDocuments({ user: resetUser!._id })).toBe(0); + expect(await db.collection('sessions').countDocuments({ user: resetUser!._id })).toBe(0); + }); + const rejected = await request.get('/api/user', { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(rejected.status()).toBe(401); + const newLogin = await request.post('/api/auth/login', { + data: { email, password: NEW_PASSWORD }, + }); + expect(newLogin.ok()).toBeTruthy(); + } finally { + try { + await Promise.all(keys.map((key) => cache.delete(key))); + } finally { + await cache.disconnect(); + await request.dispose(); + fs.rmSync(cwd, { recursive: true, force: true }); + await deleteUserByEmail(email); + } + } + }); }); diff --git a/packages/api/src/auth/credentials.eviction.spec.ts b/packages/api/src/auth/credentials.eviction.spec.ts new file mode 100644 index 0000000000..db254d9422 --- /dev/null +++ b/packages/api/src/auth/credentials.eviction.spec.ts @@ -0,0 +1,85 @@ +import { Keyv } from 'keyv'; +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { createMethods, createModels } from '@librechat/data-schemas'; +import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider'; +import type { IUser } from '@librechat/data-schemas'; +import { + getCachedAuthUserDoc, + setCachedAuthUserDoc, + buildAuthUserDocCacheKey, + AUTH_USER_DOC_CACHE_TTL_MS, +} from './userDocCache'; +import { isTokenIssuedBeforeCredentialChange } from './credentials'; + +let mongoServer: MongoMemoryServer; +const originalCacheMode = process.env.AUTH_USER_CACHE_MODE; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + createModels(mongoose); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +afterEach(() => { + if (originalCacheMode === undefined) { + delete process.env.AUTH_USER_CACHE_MODE; + } else { + process.env.AUTH_USER_CACHE_MODE = originalCacheMode; + } +}); + +describe('credential change with a failing auth cache', () => { + it('rejects an old OpenID access token once the credential barrier passes, even when the reverse index cannot be read', async () => { + process.env.AUTH_USER_CACHE_MODE = 'on'; + const store = new Keyv(); + /** The index read fails the way a throwing Redis store rejects a WRONGTYPE reply. */ + const failingStore = { + get: async (key: string) => { + if (key.startsWith(AUTH_USER_DOC_BY_ID_PREFIX)) { + throw new Error('WRONGTYPE Operation against a key holding the wrong kind of value'); + } + return store.get(key); + }, + set: (key: string, value: unknown, ttl?: number) => store.set(key, value, ttl), + delete: (key: string) => store.delete(key), + }; + const methods = createMethods(mongoose, { + getCache: (key: string) => (key === CacheKeys.AUTH_USER_DOC ? failingStore : undefined), + }); + const user = (await methods.createUser( + { email: 'oidc@example.com', provider: 'openid', openidId: 'subject-1' }, + undefined, + true, + true, + )) as Partial; + const userId = String(user._id); + const cacheKey = buildAuthUserDocCacheKey({ strategy: 'openid-jwt', subject: 'subject-1' }); + if (!cacheKey) { + throw new Error('expected a cache key'); + } + await setCachedAuthUserDoc(store, cacheKey, user); + const oldToken = { iat: Math.floor(Date.now() / 1000) - 60 }; + expect( + isTokenIssuedBeforeCredentialChange(oldToken, await getCachedAuthUserDoc(store, cacheKey)), + ).toBe(false); + + const updated = await methods.updateUser(userId, { + password: 'new-password-hash', + credentialsChangedAt: new Date(), + }); + const startedAt = Date.now(); + await methods.awaitAuthUserDocEviction(userId); + + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS); + expect(updated?.credentialsChangedAt).toBeInstanceOf(Date); + await expect(getCachedAuthUserDoc(store, cacheKey)).resolves.toBeUndefined(); + const current = await methods.findUser({ openidId: 'subject-1' }); + expect(isTokenIssuedBeforeCredentialChange(oldToken, current)).toBe(true); + }, 20_000); +}); diff --git a/packages/api/src/auth/passwordReset.ts b/packages/api/src/auth/passwordReset.ts index 3c3a2bf75b..50fc044b99 100644 --- a/packages/api/src/auth/passwordReset.ts +++ b/packages/api/src/auth/passwordReset.ts @@ -45,6 +45,8 @@ export interface PasswordResetControllerDeps { ) => Promise<{ message: string } | Error>; deleteAllUserSessions: (query: { userId: string }) => Promise<{ deletedCount?: number }>; deletePasskeysByUser: (userId: string) => Promise<{ deletedCount?: number }>; + /** Resolves once no cached auth document from before the reset can be served. */ + awaitAuthUserDocEviction: (userId: string) => Promise; } export type PasswordResetOutcome = @@ -81,6 +83,8 @@ export function createResetPasswordController(deps: PasswordResetControllerDeps) passkeys.reason, ); } + /** After the revocations, so nothing can mint a post-stamp token while it waits. */ + await deps.awaitAuthUserDocEviction(userId); return res.status(200).json(result); } catch (error) { diff --git a/packages/api/src/auth/userDocCache.spec.ts b/packages/api/src/auth/userDocCache.spec.ts index 4c70c438ab..766e33a4d9 100644 --- a/packages/api/src/auth/userDocCache.spec.ts +++ b/packages/api/src/auth/userDocCache.spec.ts @@ -13,6 +13,7 @@ import { import { cacheConfig } from '~/cache/cacheConfig'; jest.mock('@librechat/data-schemas', () => ({ + ...jest.requireActual('@librechat/data-schemas'), logger: { warn: jest.fn(), }, @@ -188,10 +189,33 @@ describe('auth user document cache helpers', () => { expect(store.set).toHaveBeenCalledWith( buildAuthUserDocReverseIndexKey(userId.toString()), [cacheKey], - AUTH_USER_DOC_CACHE_TTL_MS, + AUTH_USER_DOC_CACHE_TTL_MS * 2, ); }); + it('does not cache a document the reverse index could not record', async () => { + const store = makeStore(); + const userId = new Types.ObjectId(); + const indexKey = buildAuthUserDocReverseIndexKey(userId.toString()); + store.set.mockImplementation(async (key: string, value: unknown) => { + if (key === indexKey) { + throw new Error('redis unavailable'); + } + store.values.set(key, value); + return true; + }); + + await setCachedAuthUserDoc(store, 'auth-user-doc:v2:orphan', { + _id: userId, + email: 'user@example.com', + }); + + expect(store.values.has('auth-user-doc:v2:orphan')).toBe(false); + expect(logger.warn).toHaveBeenCalledWith('[authUserDocCache] Cache write failed', { + error: 'redis unavailable', + }); + }); + it('deduplicates reverse-index keys and caps the remembered set', async () => { const store = makeStore(); const objectId = new Types.ObjectId(); diff --git a/packages/api/src/auth/userDocCache.ts b/packages/api/src/auth/userDocCache.ts index 7a308e3823..4fa91a4772 100644 --- a/packages/api/src/auth/userDocCache.ts +++ b/packages/api/src/auth/userDocCache.ts @@ -1,11 +1,15 @@ import { createHash } from 'crypto'; -import { logger } from '@librechat/data-schemas'; -import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider'; +import { logger, evictAuthUserDocs } from '@librechat/data-schemas'; +import { + CacheKeys, + AUTH_USER_DOC_BY_ID_PREFIX, + AUTH_USER_DOC_CACHE_TTL_MS, +} from 'librechat-data-provider'; import type { IUser } from '@librechat/data-schemas'; import { cacheConfig } from '~/cache/cacheConfig'; const AUTH_USER_DOC_CACHE_VERSION = 2; -export const AUTH_USER_DOC_CACHE_TTL_MS = 5000; +export { AUTH_USER_DOC_CACHE_TTL_MS }; export type AuthUserDocCacheMode = 'off' | 'on'; @@ -164,6 +168,10 @@ export async function getCachedAuthUserDoc( } } +/** + * The reverse index is written first and outlives every document it names, so eviction can + * always find a cached document: when the index write fails, the document is not cached. + */ export async function setCachedAuthUserDoc( store: AuthUserDocCacheStore, cacheKey: string, @@ -171,6 +179,10 @@ export async function setCachedAuthUserDoc( ): Promise { try { const sanitized = sanitizeUserForCache(user); + const userId = getUserId(sanitized); + if (userId) { + await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS * 2); + } await store.set( cacheKey, { @@ -180,10 +192,6 @@ export async function setCachedAuthUserDoc( } satisfies CachedAuthUserDoc, AUTH_USER_DOC_CACHE_TTL_MS, ); - const userId = getUserId(sanitized); - if (userId) { - await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS); - } } catch (error) { logger.warn('[authUserDocCache] Cache write failed', { error: error instanceof Error ? error.message : String(error), @@ -198,25 +206,5 @@ export async function invalidateCachedAuthUserDoc( if (!store) { return; } - try { - const keys = new Set(); - if (input.cacheKey) { - keys.add(input.cacheKey); - } - if (input.userId) { - const indexKey = buildAuthUserDocReverseIndexKey(input.userId); - const indexed = await store.get(indexKey); - if (Array.isArray(indexed)) { - for (const key of indexed) { - keys.add(key); - } - } - await store.delete(indexKey); - } - await Promise.all([...keys].map((key) => store.delete(key))); - } catch (error) { - logger.warn('[authUserDocCache] Cache invalidation failed', { - error: error instanceof Error ? error.message : String(error), - }); - } + await evictAuthUserDocs(store, input); } diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts index 325bcd3e0f..dc2e77e005 100644 --- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts +++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts @@ -189,6 +189,21 @@ describe('standardCache', () => { expect(cacheWithOpts.opts.ttl).toBe(5000); }); + test('rejects failed operations only for a cache created with throwOnErrors', async () => { + const cacheFactory = await import('../../cacheFactory'); + const { ioredisClient } = await import('../../redisClients'); + if (!ioredisClient) throw new Error('ioredisClient is null'); + const strict = cacheFactory.standardCache('strict-namespace', undefined, undefined, { + throwOnErrors: true, + }); + const lenient = cacheFactory.standardCache('lenient-namespace'); + await ioredisClient.hset('strict-namespace:wrong-type', 'f', 'v'); + await ioredisClient.hset('lenient-namespace:wrong-type', 'f', 'v'); + + await expect(strict.get('wrong-type')).rejects.toThrow(/WRONGTYPE/); + await expect(lenient.get('wrong-type')).resolves.toBeUndefined(); + }); + test('should handle TTL correctly', async () => { const cacheFactory = await import('../../cacheFactory'); testCache = cacheFactory.standardCache('ttl-test', 1000); // 1 second TTL diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts index 32a1492555..0ab85a87aa 100644 --- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts +++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts @@ -67,7 +67,7 @@ describe('standardCache - CONFIG_STORE vs TOOL_CACHE namespace isolation', () => const toolCache = standardCache(CacheKeys.TOOL_CACHE); expect(MockKeyvRedis).toHaveBeenCalledTimes(1); - expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient); + expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient, { throwOnErrors: false }); expect(toolCache).toBeDefined(); }); diff --git a/packages/api/src/cache/cacheFactory.ts b/packages/api/src/cache/cacheFactory.ts index 07e443cac8..61e0d6d47a 100644 --- a/packages/api/src/cache/cacheFactory.ts +++ b/packages/api/src/cache/cacheFactory.ts @@ -85,9 +85,17 @@ async function clearRedisNamespace(namespace: string): Promise { * @param namespace - The cache namespace. * @param ttl - Time to live for cache entries. * @param fallbackStore - Optional fallback store if Redis is not used. + * @param options.throwOnErrors - Rejects failed Redis operations instead of resolving them as a + * miss or a no-op delete, for callers that must know an eviction did not happen. The first + * caller's value wins for a shared namespace and TTL. * @returns Cache instance. */ -export const standardCache = (namespace: string, ttl?: number, fallbackStore?: object): Keyv => { +export const standardCache = ( + namespace: string, + ttl?: number, + fallbackStore?: object, + options: { throwOnErrors?: boolean } = {}, +): Keyv => { if (keyvRedisClient && !cacheConfig.FORCED_IN_MEMORY_CACHE_NAMESPACES?.includes(namespace)) { const byTtl = redisCacheMap.get(namespace); const existing = byTtl?.get(ttl); @@ -95,8 +103,9 @@ export const standardCache = (namespace: string, ttl?: number, fallbackStore?: o return existing; } try { - const keyvRedis = new KeyvRedis(keyvRedisClient); - const cache = new Keyv(keyvRedis, { namespace, ttl }); + const { throwOnErrors = false } = options; + const keyvRedis = new KeyvRedis(keyvRedisClient, { throwOnErrors }); + const cache = new Keyv(keyvRedis, { namespace, ttl, throwOnErrors }); keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX; keyvRedis.keyPrefixSeparator = cacheConfig.GLOBAL_PREFIX_SEPARATOR; diff --git a/packages/api/src/favorites/pinned.spec.ts b/packages/api/src/favorites/pinned.spec.ts index 26e298db6e..c3a57e27f2 100644 --- a/packages/api/src/favorites/pinned.spec.ts +++ b/packages/api/src/favorites/pinned.spec.ts @@ -11,6 +11,7 @@ import { import { createPinnedOrderHandlers } from './pinned'; jest.mock('@librechat/data-schemas', () => ({ + ...jest.requireActual('@librechat/data-schemas'), logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() }, })); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 9386ed21b8..ae97949aab 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -4144,6 +4144,9 @@ export enum CacheKeys { export const AUTH_USER_DOC_BY_ID_PREFIX = 'auth-user-doc-byid'; +/** Lifetime of a cached auth user document, and so the longest a stale one can be served. */ +export const AUTH_USER_DOC_CACHE_TTL_MS = 5000; + /** * Enum for violation types, used to identify, log, and cache violations. */ diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index 44df9b4cd4..1ba6b2812d 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -1,6 +1,10 @@ import mongoose from 'mongoose'; import { MongoMemoryServer } from 'mongodb-memory-server'; -import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider'; +import { + CacheKeys, + AUTH_USER_DOC_BY_ID_PREFIX, + AUTH_USER_DOC_CACHE_TTL_MS, +} from 'librechat-data-provider'; import type * as t from '~/types'; import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user'; import balanceSchema from '~/schema/balance'; @@ -480,6 +484,119 @@ describe('User Methods - Database Tests', () => { expect(cache.delete).toHaveBeenCalledWith(indexKey); }); + describe('when auth cache eviction fails', () => { + const cachedKeys = ['auth-cache-key-a', 'auth-cache-key-b']; + + /** Rejects the named operations the way the throwing Redis-backed auth cache does. */ + function makeFailingCache(failing: { indexRead?: boolean; deleteKey?: string }) { + return { + get: jest.fn(async () => { + if (failing.indexRead) { + throw new Error('WRONGTYPE'); + } + return cachedKeys; + }), + set: jest.fn().mockResolvedValue(true), + delete: jest.fn(async (key: string) => { + if (key === failing.deleteKey) { + throw new Error('redis unavailable'); + } + return true; + }), + }; + } + + async function createCachedUser() { + enableAuthUserDocCache(); + const user = await User.create({ + name: 'Credential User', + email: 'credential@example.com', + provider: 'openid', + }); + return user._id?.toString() ?? ''; + } + + test.each<[string, { indexRead?: boolean; deleteKey?: string }]>([ + ['the reverse index read fails', { indexRead: true }], + ['an indexed document delete fails', { deleteKey: 'auth-cache-key-a' }], + ])('waits out the cache TTL at the credential barrier when %s', async (_label, failing) => { + const userId = await createCachedUser(); + const cache = makeFailingCache(failing); + const delay = jest.fn().mockResolvedValue(undefined); + const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay }); + + const updated = await methodsWithCache.updateUser(userId, { + password: 'new-password-hash', + credentialsChangedAt: new Date(), + }); + expect(updated?.credentialsChangedAt).toBeInstanceOf(Date); + expect(delay).not.toHaveBeenCalled(); + + await methodsWithCache.awaitAuthUserDocEviction(userId); + + expect(delay).toHaveBeenCalledTimes(1); + expect(delay.mock.calls[0][0]).toBeGreaterThan(AUTH_USER_DOC_CACHE_TTL_MS); + expect(cache.delete).not.toHaveBeenCalledWith(`${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`); + if (!failing.indexRead) { + expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b'); + } + }); + + test('passes the barrier at once when only the index delete fails', async () => { + const userId = await createCachedUser(); + const cache = makeFailingCache({ deleteKey: `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}` }); + const delay = jest.fn().mockResolvedValue(undefined); + const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay }); + + await methodsWithCache.awaitAuthUserDocEviction(userId); + + expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a'); + expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b'); + expect(delay).not.toHaveBeenCalled(); + }); + + test('passes the barrier at once when a retried eviction succeeds', async () => { + const userId = await createCachedUser(); + const cache = makeFailingCache({ indexRead: true }); + const delay = jest.fn().mockResolvedValue(undefined); + const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay }); + + await methodsWithCache.updateUser(userId, { credentialsChangedAt: new Date() }); + cache.get.mockResolvedValue(cachedKeys); + await methodsWithCache.awaitAuthUserDocEviction(userId); + + expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a'); + expect(delay).not.toHaveBeenCalled(); + }); + + test('waits at the barrier when the cache can read but not delete', async () => { + const userId = await createCachedUser(); + const delay = jest.fn().mockResolvedValue(undefined); + const methodsWithCache = createUserMethods(mongoose, { + getCache: () => ({ get: jest.fn(), set: jest.fn() }), + delay, + }); + + await methodsWithCache.awaitAuthUserDocEviction(userId); + + expect(delay).toHaveBeenCalledTimes(1); + }); + + test('keeps updates themselves best effort', async () => { + const userId = await createCachedUser(); + const delay = jest.fn().mockResolvedValue(undefined); + const methodsWithCache = createUserMethods(mongoose, { + getCache: () => makeFailingCache({ indexRead: true }), + delay, + }); + + const updated = await methodsWithCache.updateUser(userId, { name: 'Renamed' }); + + expect(updated?.name).toBe('Renamed'); + expect(delay).not.toHaveBeenCalled(); + }); + }); + test('should invalidate cached auth user documents on delete', async () => { enableAuthUserDocCache(); const user = await User.create({ diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index 07de2ded81..cfa9057a10 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -1,14 +1,16 @@ import mongoose, { FilterQuery } from 'mongoose'; import { - AUTH_USER_DOC_BY_ID_PREFIX, + AUTH_USER_DOC_CACHE_TTL_MS, CacheKeys, type RefillIntervalUnit, type StatefulCodeEnvironment, } from 'librechat-data-provider'; import type { IUser, BalanceConfig, CreateUserRequest, UserDeleteResult } from '~/types'; import type { CacheStore } from '~/types'; +import { evictAuthUserDocs } from '~/utils/eviction'; import { escapeRegExp } from '~/utils/string'; import { signPayload } from '~/crypto'; +import logger from '~/config/winston'; /** Default JWT session expiry: 15 minutes in milliseconds */ export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15; @@ -19,8 +21,15 @@ const MAX_SUBAGENT_ADMISSION_FENCES = 32; interface UserMethodDeps { getCache?: (key: string) => CacheStore | undefined; + /** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */ + delay?: (ms: number) => Promise; } +const wait = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Timers can fire a millisecond early, so the wait outlasts the cache TTL by this much. */ +const AUTH_USER_DOC_EXPIRY_MARGIN_MS = 100; + function isAuthUserDocCacheEnabled(): boolean { return process.env.AUTH_USER_CACHE_MODE === 'on'; } @@ -55,6 +64,7 @@ export function createUserMethods( expectedState?: FilterQuery, options?: { preserveExpiresAt?: boolean }, ) => Promise; + awaitAuthUserDocEviction: (userId: string) => Promise; claimSamlIdentity: ( userId: string, samlId: string, @@ -333,6 +343,24 @@ export function createUserMethods( return updated; } + /** + * The barrier a credential change passes before it is confirmed. A cached document without + * the new credentialsChangedAt keeps pre-change access tokens verifying until it expires, so + * eviction is retried and, when it still cannot remove every document, this resolves only + * after the cache TTL. Callers revoke sessions and passkeys first: during the wait those + * could otherwise mint access tokens issued after the stamp. + */ + async function awaitAuthUserDocEviction(userId: string): Promise { + if (await invalidateAuthUserDocCache(userId)) { + return; + } + logger.warn( + '[awaitAuthUserDocEviction] Cached auth documents were not evicted after a credential change; waiting for them to expire', + { userId, waitMs: AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS }, + ); + await (deps.delay ?? wait)(AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS); + } + /** Atomically updates a SAML user only when the incoming identity can claim the document. */ async function claimSamlIdentity( userId: string, @@ -358,26 +386,20 @@ export function createUserMethods( return updated; } - async function invalidateAuthUserDocCache(userId: string): Promise { + /** Resolves false only when a cached document for the user may still be served. */ + async function invalidateAuthUserDocCache(userId: string): Promise { if (!isAuthUserDocCacheEnabled()) { - return; + return true; } const cache = deps.getCache?.(CacheKeys.AUTH_USER_DOC); - if (!cache?.get || !cache?.delete) { - return; + if (!cache?.get) { + return true; } - try { - const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`; - const cachedKeys = await cache.get(indexKey); - if (Array.isArray(cachedKeys)) { - await Promise.all( - cachedKeys.map((key) => (typeof key === 'string' ? cache.delete?.(key) : undefined)), - ); - } - await cache.delete(indexKey); - } catch { - // Cache invalidation must not make a user update fail. + const remove = cache.delete?.bind(cache); + if (!remove) { + return false; } + return evictAuthUserDocs({ get: (key) => cache.get(key), delete: remove }, { userId }); } /** @@ -863,6 +885,7 @@ export function createUserMethods( countUsers, createUser, updateUser, + awaitAuthUserDocEviction, claimSamlIdentity, acceptTerms, searchUsers, diff --git a/packages/data-schemas/src/utils/eviction.spec.ts b/packages/data-schemas/src/utils/eviction.spec.ts new file mode 100644 index 0000000000..4114788ab5 --- /dev/null +++ b/packages/data-schemas/src/utils/eviction.spec.ts @@ -0,0 +1,111 @@ +import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider'; +import { evictAuthUserDocs } from './eviction'; + +const INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-1`; +const OTHER_INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-2`; + +/** A Map-backed store that rejects the operations named for a key, as a throwing Redis store does. */ +function makeStore(failures: { get?: string[]; delete?: string[] } = {}) { + const values = new Map([ + [INDEX_KEY, ['doc-a', 'doc-b']], + ['doc-a', { id: 'user-1' }], + ['doc-b', { id: 'user-1' }], + ['doc-c', { id: 'user-1' }], + [OTHER_INDEX_KEY, ['doc-other']], + ['doc-other', { id: 'user-2' }], + ]); + return { + values, + get: jest.fn(async (key: string) => { + if (failures.get?.includes(key)) { + throw new Error(`GET ${key} failed`); + } + return values.get(key); + }), + delete: jest.fn(async (key: string) => { + if (failures.delete?.includes(key)) { + throw new Error(`DEL ${key} failed`); + } + return values.delete(key); + }), + }; +} + +function expectOtherUserCached(store: ReturnType) { + expect(store.values.get(OTHER_INDEX_KEY)).toEqual(['doc-other']); + expect(store.values.get('doc-other')).toEqual({ id: 'user-2' }); +} + +describe('evictAuthUserDocs', () => { + it('lets a retry find a document an earlier failed delete left behind', async () => { + const store = makeStore({ delete: ['doc-a'] }); + await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false); + + store.delete.mockImplementation(async (key: string) => store.values.delete(key)); + await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true); + + expect(store.values.has('doc-a')).toBe(false); + expect(store.values.has(INDEX_KEY)).toBe(false); + }); + + it('deletes the indexed and explicit documents and the index', async () => { + const store = makeStore(); + + await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe( + true, + ); + + for (const key of [INDEX_KEY, 'doc-a', 'doc-b', 'doc-c']) { + expect(store.values.has(key)).toBe(false); + } + expectOtherUserCached(store); + }); + + it('reports a failed index read and keeps the index for a retry', async () => { + const store = makeStore({ get: [INDEX_KEY] }); + + await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe( + false, + ); + + expect(store.values.has('doc-c')).toBe(false); + expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']); + expect(store.values.has('doc-a')).toBe(true); + expectOtherUserCached(store); + }); + + it.each([ + ['a string', 'doc-a'], + ['a list with a non-string entry', ['doc-a', 42]], + ])('treats a reverse index holding %s as unreadable', async (_label, indexed) => { + const store = makeStore(); + store.values.set(INDEX_KEY, indexed); + + await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false); + + expect(store.values.get(INDEX_KEY)).toEqual(indexed); + expect(store.values.has('doc-a')).toBe(true); + }); + + it('reports a failed document delete, deletes the others and keeps the index for a retry', async () => { + const store = makeStore({ delete: ['doc-a'] }); + + await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false); + + expect(store.values.has('doc-a')).toBe(true); + expect(store.values.has('doc-b')).toBe(false); + expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']); + expectOtherUserCached(store); + }); + + it('counts documents as evicted when only the index delete fails', async () => { + const store = makeStore({ delete: [INDEX_KEY] }); + + await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true); + + expect(store.values.has('doc-a')).toBe(false); + expect(store.values.has('doc-b')).toBe(false); + expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']); + expectOtherUserCached(store); + }); +}); diff --git a/packages/data-schemas/src/utils/eviction.ts b/packages/data-schemas/src/utils/eviction.ts new file mode 100644 index 0000000000..f238b38a32 --- /dev/null +++ b/packages/data-schemas/src/utils/eviction.ts @@ -0,0 +1,77 @@ +import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider'; +import logger from '~/config/winston'; + +/** Store operations eviction needs; a failure is only seen when the store throws it. */ +export interface AuthUserDocEvictionStore { + get: (key: string) => Promise; + delete: (key: string) => Promise; +} + +function isKeyList(value: unknown): value is string[] { + return Array.isArray(value) && value.every((key) => typeof key === 'string'); +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Deletes the cached auth user documents for a user, attempting every delete even when + * another fails. Resolves false when a document may still be served: the reverse index could + * not be read, so its keys are unknown, or a document delete failed. The index is deleted only + * after every document it names is gone, so a retry can still find what an earlier attempt + * left behind. A failed delete of the index alone leaves only the names of deleted keys, so it + * is logged and still counts as evicted. Never throws, so callers keep their own write results. + */ +export async function evictAuthUserDocs( + store: AuthUserDocEvictionStore, + input: { userId?: string; cacheKey?: string }, +): Promise { + const keys = new Set(input.cacheKey ? [input.cacheKey] : []); + const indexKey = input.userId ? `${AUTH_USER_DOC_BY_ID_PREFIX}:${input.userId}` : undefined; + let indexRead = true; + if (indexKey) { + try { + const indexed = await store.get(indexKey); + if (indexed != null && !isKeyList(indexed)) { + throw new Error('Reverse index is not a list of cache keys'); + } + for (const key of indexed ?? []) { + keys.add(key); + } + } catch (error) { + indexRead = false; + logger.warn('[authUserDocCache] Reverse index read failed during eviction', { + userId: input.userId, + error: describeError(error), + }); + } + } + + const results = await Promise.allSettled([...keys].map((key) => store.delete(key))); + let documentsDeleted = true; + for (const result of results) { + if (result.status === 'rejected') { + documentsDeleted = false; + logger.warn('[authUserDocCache] Cached document delete failed during eviction', { + userId: input.userId, + error: describeError(result.reason), + }); + } + } + if (!indexRead || !documentsDeleted) { + return false; + } + + if (indexKey) { + try { + await store.delete(indexKey); + } catch (error) { + logger.warn('[authUserDocCache] Reverse index delete failed during eviction', { + userId: input.userId, + error: describeError(error), + }); + } + } + return true; +} diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts index fecdfcc7ed..f5a4c6fd48 100644 --- a/packages/data-schemas/src/utils/index.ts +++ b/packages/data-schemas/src/utils/index.ts @@ -8,5 +8,6 @@ export * from './objectId'; export * from './yaml'; export * from './stripUIResourceMarkers'; export * from './fading'; +export * from './eviction'; export { buildIndexWithRetry, createIndexesWithRetry, isIndexBuildInProgress } from './retry'; export type { IndexBuildOptions } from './retry';