mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🧯 fix: Outlast Failed Auth Cache Eviction on Credential Changes (#16435)
* fix: outlast failed auth cache eviction on credential changes A credential change committed to MongoDB could report success while a cached auth user document without the new credentialsChangedAt stayed readable, so an old OpenID access token kept authorizing until the cache TTL. The Redis-backed auth cache swallowed every failure (Keyv and KeyvRedis resolve a failed GET as a miss and a failed DEL as a no-op), and the eviction helpers stopped at the first error. - The AUTH_USER_DOC namespace opts into throwOnErrors so a failed read or delete is visible; every caller of that namespace already catches. - evictAuthUserDocs attempts every delete even when another fails and reports whether a document may still be served; the data-schemas and packages/api invalidation paths share it. - updateUser waits out AUTH_USER_DOC_CACHE_TTL_MS before returning a credential change whose eviction was incomplete, so the change is only confirmed once no pre-change document can be served. Other updates stay best effort, and later revocations are delayed, never skipped. * fix: record the reverse index before caching an auth user document A document written before its reverse-index entry was cached without an index naming it when the index write failed, so eviction could not find it. Write the index first with a TTL that outlives the documents it names, and skip the document when the index write fails. * test: keep real data-schemas helpers in the pinned order spec * fix: settle the auth cache after revoking reset credentials Waiting out the cache TTL inside updateUser held the reset before its session and passkey revocations, so a live refresh token or passkey could mint an access token issued after credentialsChangedAt during the wait. updateUser is best effort again; awaitAuthUserDocEviction retries the eviction and waits one TTL only when it is still incomplete, and both the web reset controller and the reset CLI call it after revoking sessions and passkeys, before confirming the reset. * fix: settle the credential barrier with a margin and on CLI revocation errors Timers can fire a millisecond early, so the eviction barrier now waits the cache TTL plus a 100 ms margin. The reset CLI also settles the barrier when revoking passkeys or sessions throws, since the password is already changed. * fix: treat an auth cache that cannot delete as not evicted * fix: keep the reverse index until every document it names is evicted A failed eviction still deleted the reverse index, so the credential barrier's retry found no index and reported the cache clear while the documents an unreadable index or a failed delete left behind stayed cached. The index is now deleted only after it was read and every document it names is gone, so a retry can still find what remains. * fix: treat a malformed auth reverse index as unreadable * fix: treat a reverse index with a non-string entry as unreadable
This commit is contained in:
parent
9ac96c5ca9
commit
18b6937da3
19 changed files with 655 additions and 57 deletions
4
api/cache/getLogStores.js
vendored
4
api/cache/getLogStores.js
vendored
|
|
@ -66,7 +66,9 @@ const namespaces = {
|
|||
CacheKeys.OPENID_EXCHANGED_TOKENS,
|
||||
Time.TEN_MINUTES,
|
||||
),
|
||||
[CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC),
|
||||
[CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC, undefined, undefined, {
|
||||
throwOnErrors: true,
|
||||
}),
|
||||
[CacheKeys.ADMIN_OAUTH_EXCHANGE]: standardCache(
|
||||
CacheKeys.ADMIN_OAUTH_EXCHANGE,
|
||||
Time.THIRTY_SECONDS,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const {
|
|||
const {
|
||||
deleteAllUserSessions,
|
||||
deletePasskeysByUser,
|
||||
awaitAuthUserDocEviction,
|
||||
getUserById,
|
||||
findSession,
|
||||
updateUser,
|
||||
|
|
@ -326,6 +327,7 @@ const resetPasswordController = createResetPasswordController({
|
|||
resetPassword,
|
||||
deleteAllUserSessions,
|
||||
deletePasskeysByUser,
|
||||
awaitAuthUserDocEviction,
|
||||
});
|
||||
|
||||
const refreshController = async (req, res) => {
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ jest.mock('~/models', () => ({
|
|||
deleteSession: jest.fn(),
|
||||
deleteAllUserSessions: jest.fn(),
|
||||
deletePasskeysByUser: jest.fn(),
|
||||
awaitAuthUserDocEviction: jest.fn(),
|
||||
getUserById: jest.fn(),
|
||||
findSession: jest.fn(),
|
||||
updateUser: jest.fn(),
|
||||
|
|
@ -147,6 +148,7 @@ const {
|
|||
deleteTokens,
|
||||
deleteAllUserSessions,
|
||||
deletePasskeysByUser,
|
||||
awaitAuthUserDocEviction,
|
||||
} = require('~/models');
|
||||
const {
|
||||
createRefreshTokenBridgeFlightKey,
|
||||
|
|
@ -2483,6 +2485,7 @@ describe('resetPasswordController', () => {
|
|||
resetPassword.mockReset().mockResolvedValue(serviceResult);
|
||||
deleteAllUserSessions.mockReset().mockResolvedValue(undefined);
|
||||
deletePasskeysByUser.mockReset().mockResolvedValue(undefined);
|
||||
awaitAuthUserDocEviction.mockReset().mockResolvedValue(undefined);
|
||||
req = {
|
||||
body: {
|
||||
userId: 'user-123',
|
||||
|
|
@ -2511,6 +2514,31 @@ describe('resetPasswordController', () => {
|
|||
expect(res.json).toHaveBeenCalledWith(serviceResult);
|
||||
});
|
||||
|
||||
it('confirms the reset only after revoking and settling the auth cache', async () => {
|
||||
let finishEviction;
|
||||
awaitAuthUserDocEviction.mockReturnValue(
|
||||
new Promise((resolve) => {
|
||||
finishEviction = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const reset = resetPasswordController(req, res);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const responseCalls = res.status.mock.calls.length;
|
||||
finishEviction();
|
||||
await reset;
|
||||
|
||||
expect(awaitAuthUserDocEviction).toHaveBeenCalledWith('user-123');
|
||||
expect(deleteAllUserSessions.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
awaitAuthUserDocEviction.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(deletePasskeysByUser.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
awaitAuthUserDocEviction.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(responseCalls).toBe(0);
|
||||
expect(res.status).toHaveBeenCalledWith(200);
|
||||
});
|
||||
|
||||
it('does not revoke sessions or passkeys when reset fails', async () => {
|
||||
const resetError = new Error('Invalid token');
|
||||
resetPassword.mockResolvedValue(resetError);
|
||||
|
|
@ -2519,6 +2547,7 @@ describe('resetPasswordController', () => {
|
|||
|
||||
expect(deleteAllUserSessions).not.toHaveBeenCalled();
|
||||
expect(deletePasskeysByUser).not.toHaveBeenCalled();
|
||||
expect(awaitAuthUserDocEviction).not.toHaveBeenCalled();
|
||||
expect(res.status).toHaveBeenCalledWith(400);
|
||||
expect(res.json).toHaveBeenCalledWith(resetError);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -85,9 +85,15 @@ const resetPassword = async () => {
|
|||
* A passkey signs in on its own, so leaving one in place would keep an attacker
|
||||
* logged in after an administrator believes the account has been recovered.
|
||||
*/
|
||||
const { deletedCount } = await Passkey.deleteMany({ user: user._id });
|
||||
/** A refresh session outlives the stamp: refreshing mints a token issued after it. */
|
||||
await Session.deleteMany({ user: user._id });
|
||||
let deletedCount;
|
||||
try {
|
||||
({ deletedCount } = await Passkey.deleteMany({ user: user._id }));
|
||||
/** A refresh session outlives the stamp: refreshing mints a token issued after it. */
|
||||
await Session.deleteMany({ user: user._id });
|
||||
} finally {
|
||||
/** The password is already changed, so the cache settles even when a revocation fails. */
|
||||
await methods.awaitAuthUserDocEviction(user._id.toString());
|
||||
}
|
||||
|
||||
console.log('Password successfully reset!');
|
||||
if (deletedCount > 0) {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,11 @@ import os from 'os';
|
|||
import { Keyv } from 'keyv';
|
||||
import KeyvRedis from '@keyv/redis';
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
|
||||
import {
|
||||
CacheKeys,
|
||||
AUTH_USER_DOC_BY_ID_PREFIX,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
} from 'librechat-data-provider';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { randomInt, randomUUID } from 'node:crypto';
|
||||
import { withMongo, seedPasskey, deleteUserByEmail } from '../db';
|
||||
|
|
@ -40,7 +44,13 @@ function runResetCli(
|
|||
email: string,
|
||||
cwd: string,
|
||||
env: NodeJS.ProcessEnv,
|
||||
): Promise<{ code: number | null; stdout: string; stderr: string }> {
|
||||
): Promise<{
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
/** Milliseconds between confirming the new password and the CLI reporting success. */
|
||||
reportDelayMs?: number;
|
||||
}> {
|
||||
const child = spawn(process.execPath, [path.join(REPO_ROOT, 'config/reset-password.js')], {
|
||||
cwd,
|
||||
env: {
|
||||
|
|
@ -59,14 +69,20 @@ function runResetCli(
|
|||
['Confirm new password: ', NEW_PASSWORD],
|
||||
];
|
||||
let stdout = '';
|
||||
let confirmedAt: number | undefined;
|
||||
let reportedAt: number | undefined;
|
||||
child.stdout?.on('data', (chunk: Buffer) => {
|
||||
stdout += chunk.toString();
|
||||
answers.forEach((answer, index) => {
|
||||
if (answer !== null && stdout.includes(answer[0])) {
|
||||
child.stdin?.write(`${answer[1]}\n`);
|
||||
answers[index] = null;
|
||||
confirmedAt = Date.now();
|
||||
}
|
||||
});
|
||||
if (reportedAt === undefined && stdout.includes('Password successfully reset!')) {
|
||||
reportedAt = Date.now();
|
||||
}
|
||||
});
|
||||
let stderr = '';
|
||||
child.stderr?.on('data', (chunk: Buffer) => (stderr += chunk.toString()));
|
||||
|
|
@ -82,7 +98,11 @@ function runResetCli(
|
|||
child.on('close', (code) => {
|
||||
clearTimeout(timer);
|
||||
child.stdin?.end();
|
||||
resolve({ code, stdout, stderr });
|
||||
const reportDelayMs =
|
||||
confirmedAt !== undefined && reportedAt !== undefined
|
||||
? reportedAt - confirmedAt
|
||||
: undefined;
|
||||
resolve({ code, stdout, stderr, reportDelayMs });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -230,4 +250,85 @@ test.describe('CLI password reset', () => {
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('reports the reset only after unevictable cached credentials expire @scenario:cli-reset-outlasts-failed-cache-eviction', async ({
|
||||
playwright,
|
||||
baseURL,
|
||||
}) => {
|
||||
test.setTimeout(90_000);
|
||||
const request = await playwright.request.newContext({
|
||||
baseURL,
|
||||
extraHTTPHeaders: {
|
||||
'X-Forwarded-For': `127.${randomInt(1, 255)}.${randomInt(1, 255)}.${randomInt(1, 255)}`,
|
||||
},
|
||||
});
|
||||
const email = `cli-evict-${randomUUID().slice(0, 8)}@example.com`;
|
||||
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-reset-'));
|
||||
const redisUri = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379';
|
||||
const prefix = `cli-evict-${randomUUID()}`;
|
||||
const adapter = new KeyvRedis(redisUri, { throwOnErrors: true, connectionTimeout: 5000 });
|
||||
const cache = new Keyv(adapter, { namespace: CacheKeys.AUTH_USER_DOC });
|
||||
adapter.namespace = prefix;
|
||||
adapter.keyPrefixSeparator = '::';
|
||||
const keys: string[] = [];
|
||||
try {
|
||||
const register = await request.post('/api/auth/register', {
|
||||
data: { email, name: 'CLI Evict', password: PASSWORD, confirm_password: PASSWORD },
|
||||
});
|
||||
expect(register.ok()).toBeTruthy();
|
||||
const login = await request.post('/api/auth/login', { data: { email, password: PASSWORD } });
|
||||
expect(login.ok()).toBeTruthy();
|
||||
const { token, user } = (await login.json()) as { token: string; user: { _id: string } };
|
||||
await seedPasskey(email, randomUUID(), 'Eviction regression');
|
||||
|
||||
const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${user._id}`;
|
||||
const controlKey = 'auth-user-doc:another-user';
|
||||
keys.push(indexKey, controlKey);
|
||||
await cache.set(controlKey, { credentialsChangedAt: null }, 120_000);
|
||||
/** A hash where the index string belongs makes the CLI's GET fail with WRONGTYPE. */
|
||||
const client = await adapter.getClient();
|
||||
await client.hSet(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 'unreadable', '1');
|
||||
await client.pExpire(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 120_000);
|
||||
|
||||
const env = { ...process.env };
|
||||
Object.assign(env, {
|
||||
USE_REDIS: 'true',
|
||||
USE_REDIS_CLUSTER: 'false',
|
||||
REDIS_URI: redisUri,
|
||||
REDIS_KEY_PREFIX: prefix,
|
||||
REDIS_KEY_PREFIX_VAR: '',
|
||||
AUTH_USER_CACHE_MODE: 'on',
|
||||
FORCED_IN_MEMORY_CACHE_NAMESPACES: '',
|
||||
});
|
||||
|
||||
const cli = await runResetCli(email, cwd, env);
|
||||
expect(cli.code, cli.stderr.slice(-400)).toBe(0);
|
||||
expect(cli.stdout).toContain('Password successfully reset!');
|
||||
expect(cli.reportDelayMs).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS);
|
||||
expect(await cache.get(controlKey)).toEqual({ credentialsChangedAt: null });
|
||||
await withMongo(async (db) => {
|
||||
const resetUser = await db.collection('users').findOne({ email });
|
||||
expect(resetUser?.credentialsChangedAt).toBeTruthy();
|
||||
expect(await db.collection('passkeys').countDocuments({ user: resetUser!._id })).toBe(0);
|
||||
expect(await db.collection('sessions').countDocuments({ user: resetUser!._id })).toBe(0);
|
||||
});
|
||||
const rejected = await request.get('/api/user', {
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
});
|
||||
expect(rejected.status()).toBe(401);
|
||||
const newLogin = await request.post('/api/auth/login', {
|
||||
data: { email, password: NEW_PASSWORD },
|
||||
});
|
||||
expect(newLogin.ok()).toBeTruthy();
|
||||
} finally {
|
||||
try {
|
||||
await Promise.all(keys.map((key) => cache.delete(key)));
|
||||
} finally {
|
||||
await cache.disconnect();
|
||||
await request.dispose();
|
||||
fs.rmSync(cwd, { recursive: true, force: true });
|
||||
await deleteUserByEmail(email);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
85
packages/api/src/auth/credentials.eviction.spec.ts
Normal file
85
packages/api/src/auth/credentials.eviction.spec.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { Keyv } from 'keyv';
|
||||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { createMethods, createModels } from '@librechat/data-schemas';
|
||||
import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
|
||||
import type { IUser } from '@librechat/data-schemas';
|
||||
import {
|
||||
getCachedAuthUserDoc,
|
||||
setCachedAuthUserDoc,
|
||||
buildAuthUserDocCacheKey,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
} from './userDocCache';
|
||||
import { isTokenIssuedBeforeCredentialChange } from './credentials';
|
||||
|
||||
let mongoServer: MongoMemoryServer;
|
||||
const originalCacheMode = process.env.AUTH_USER_CACHE_MODE;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongoServer = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongoServer.getUri());
|
||||
createModels(mongoose);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongoServer.stop();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalCacheMode === undefined) {
|
||||
delete process.env.AUTH_USER_CACHE_MODE;
|
||||
} else {
|
||||
process.env.AUTH_USER_CACHE_MODE = originalCacheMode;
|
||||
}
|
||||
});
|
||||
|
||||
describe('credential change with a failing auth cache', () => {
|
||||
it('rejects an old OpenID access token once the credential barrier passes, even when the reverse index cannot be read', async () => {
|
||||
process.env.AUTH_USER_CACHE_MODE = 'on';
|
||||
const store = new Keyv();
|
||||
/** The index read fails the way a throwing Redis store rejects a WRONGTYPE reply. */
|
||||
const failingStore = {
|
||||
get: async (key: string) => {
|
||||
if (key.startsWith(AUTH_USER_DOC_BY_ID_PREFIX)) {
|
||||
throw new Error('WRONGTYPE Operation against a key holding the wrong kind of value');
|
||||
}
|
||||
return store.get(key);
|
||||
},
|
||||
set: (key: string, value: unknown, ttl?: number) => store.set(key, value, ttl),
|
||||
delete: (key: string) => store.delete(key),
|
||||
};
|
||||
const methods = createMethods(mongoose, {
|
||||
getCache: (key: string) => (key === CacheKeys.AUTH_USER_DOC ? failingStore : undefined),
|
||||
});
|
||||
const user = (await methods.createUser(
|
||||
{ email: 'oidc@example.com', provider: 'openid', openidId: 'subject-1' },
|
||||
undefined,
|
||||
true,
|
||||
true,
|
||||
)) as Partial<IUser>;
|
||||
const userId = String(user._id);
|
||||
const cacheKey = buildAuthUserDocCacheKey({ strategy: 'openid-jwt', subject: 'subject-1' });
|
||||
if (!cacheKey) {
|
||||
throw new Error('expected a cache key');
|
||||
}
|
||||
await setCachedAuthUserDoc(store, cacheKey, user);
|
||||
const oldToken = { iat: Math.floor(Date.now() / 1000) - 60 };
|
||||
expect(
|
||||
isTokenIssuedBeforeCredentialChange(oldToken, await getCachedAuthUserDoc(store, cacheKey)),
|
||||
).toBe(false);
|
||||
|
||||
const updated = await methods.updateUser(userId, {
|
||||
password: 'new-password-hash',
|
||||
credentialsChangedAt: new Date(),
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
await methods.awaitAuthUserDocEviction(userId);
|
||||
|
||||
expect(Date.now() - startedAt).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS);
|
||||
expect(updated?.credentialsChangedAt).toBeInstanceOf(Date);
|
||||
await expect(getCachedAuthUserDoc(store, cacheKey)).resolves.toBeUndefined();
|
||||
const current = await methods.findUser({ openidId: 'subject-1' });
|
||||
expect(isTokenIssuedBeforeCredentialChange(oldToken, current)).toBe(true);
|
||||
}, 20_000);
|
||||
});
|
||||
|
|
@ -45,6 +45,8 @@ export interface PasswordResetControllerDeps {
|
|||
) => Promise<{ message: string } | Error>;
|
||||
deleteAllUserSessions: (query: { userId: string }) => Promise<{ deletedCount?: number }>;
|
||||
deletePasskeysByUser: (userId: string) => Promise<{ deletedCount?: number }>;
|
||||
/** Resolves once no cached auth document from before the reset can be served. */
|
||||
awaitAuthUserDocEviction: (userId: string) => Promise<void>;
|
||||
}
|
||||
|
||||
export type PasswordResetOutcome =
|
||||
|
|
@ -81,6 +83,8 @@ export function createResetPasswordController(deps: PasswordResetControllerDeps)
|
|||
passkeys.reason,
|
||||
);
|
||||
}
|
||||
/** After the revocations, so nothing can mint a post-stamp token while it waits. */
|
||||
await deps.awaitAuthUserDocEviction(userId);
|
||||
|
||||
return res.status(200).json(result);
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
import { cacheConfig } from '~/cache/cacheConfig';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
...jest.requireActual('@librechat/data-schemas'),
|
||||
logger: {
|
||||
warn: jest.fn(),
|
||||
},
|
||||
|
|
@ -188,10 +189,33 @@ describe('auth user document cache helpers', () => {
|
|||
expect(store.set).toHaveBeenCalledWith(
|
||||
buildAuthUserDocReverseIndexKey(userId.toString()),
|
||||
[cacheKey],
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS * 2,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not cache a document the reverse index could not record', async () => {
|
||||
const store = makeStore();
|
||||
const userId = new Types.ObjectId();
|
||||
const indexKey = buildAuthUserDocReverseIndexKey(userId.toString());
|
||||
store.set.mockImplementation(async (key: string, value: unknown) => {
|
||||
if (key === indexKey) {
|
||||
throw new Error('redis unavailable');
|
||||
}
|
||||
store.values.set(key, value);
|
||||
return true;
|
||||
});
|
||||
|
||||
await setCachedAuthUserDoc(store, 'auth-user-doc:v2:orphan', {
|
||||
_id: userId,
|
||||
email: 'user@example.com',
|
||||
});
|
||||
|
||||
expect(store.values.has('auth-user-doc:v2:orphan')).toBe(false);
|
||||
expect(logger.warn).toHaveBeenCalledWith('[authUserDocCache] Cache write failed', {
|
||||
error: 'redis unavailable',
|
||||
});
|
||||
});
|
||||
|
||||
it('deduplicates reverse-index keys and caps the remembered set', async () => {
|
||||
const store = makeStore();
|
||||
const objectId = new Types.ObjectId();
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
import { createHash } from 'crypto';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
|
||||
import { logger, evictAuthUserDocs } from '@librechat/data-schemas';
|
||||
import {
|
||||
CacheKeys,
|
||||
AUTH_USER_DOC_BY_ID_PREFIX,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
} from 'librechat-data-provider';
|
||||
import type { IUser } from '@librechat/data-schemas';
|
||||
import { cacheConfig } from '~/cache/cacheConfig';
|
||||
|
||||
const AUTH_USER_DOC_CACHE_VERSION = 2;
|
||||
export const AUTH_USER_DOC_CACHE_TTL_MS = 5000;
|
||||
export { AUTH_USER_DOC_CACHE_TTL_MS };
|
||||
|
||||
export type AuthUserDocCacheMode = 'off' | 'on';
|
||||
|
||||
|
|
@ -164,6 +168,10 @@ export async function getCachedAuthUserDoc(
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The reverse index is written first and outlives every document it names, so eviction can
|
||||
* always find a cached document: when the index write fails, the document is not cached.
|
||||
*/
|
||||
export async function setCachedAuthUserDoc(
|
||||
store: AuthUserDocCacheStore,
|
||||
cacheKey: string,
|
||||
|
|
@ -171,6 +179,10 @@ export async function setCachedAuthUserDoc(
|
|||
): Promise<void> {
|
||||
try {
|
||||
const sanitized = sanitizeUserForCache(user);
|
||||
const userId = getUserId(sanitized);
|
||||
if (userId) {
|
||||
await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS * 2);
|
||||
}
|
||||
await store.set(
|
||||
cacheKey,
|
||||
{
|
||||
|
|
@ -180,10 +192,6 @@ export async function setCachedAuthUserDoc(
|
|||
} satisfies CachedAuthUserDoc,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
);
|
||||
const userId = getUserId(sanitized);
|
||||
if (userId) {
|
||||
await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('[authUserDocCache] Cache write failed', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
|
|
@ -198,25 +206,5 @@ export async function invalidateCachedAuthUserDoc(
|
|||
if (!store) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const keys = new Set<string>();
|
||||
if (input.cacheKey) {
|
||||
keys.add(input.cacheKey);
|
||||
}
|
||||
if (input.userId) {
|
||||
const indexKey = buildAuthUserDocReverseIndexKey(input.userId);
|
||||
const indexed = await store.get<string[]>(indexKey);
|
||||
if (Array.isArray(indexed)) {
|
||||
for (const key of indexed) {
|
||||
keys.add(key);
|
||||
}
|
||||
}
|
||||
await store.delete(indexKey);
|
||||
}
|
||||
await Promise.all([...keys].map((key) => store.delete(key)));
|
||||
} catch (error) {
|
||||
logger.warn('[authUserDocCache] Cache invalidation failed', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
await evictAuthUserDocs(store, input);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -189,6 +189,21 @@ describe('standardCache', () => {
|
|||
expect(cacheWithOpts.opts.ttl).toBe(5000);
|
||||
});
|
||||
|
||||
test('rejects failed operations only for a cache created with throwOnErrors', async () => {
|
||||
const cacheFactory = await import('../../cacheFactory');
|
||||
const { ioredisClient } = await import('../../redisClients');
|
||||
if (!ioredisClient) throw new Error('ioredisClient is null');
|
||||
const strict = cacheFactory.standardCache('strict-namespace', undefined, undefined, {
|
||||
throwOnErrors: true,
|
||||
});
|
||||
const lenient = cacheFactory.standardCache('lenient-namespace');
|
||||
await ioredisClient.hset('strict-namespace:wrong-type', 'f', 'v');
|
||||
await ioredisClient.hset('lenient-namespace:wrong-type', 'f', 'v');
|
||||
|
||||
await expect(strict.get('wrong-type')).rejects.toThrow(/WRONGTYPE/);
|
||||
await expect(lenient.get('wrong-type')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('should handle TTL correctly', async () => {
|
||||
const cacheFactory = await import('../../cacheFactory');
|
||||
testCache = cacheFactory.standardCache('ttl-test', 1000); // 1 second TTL
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ describe('standardCache - CONFIG_STORE vs TOOL_CACHE namespace isolation', () =>
|
|||
|
||||
const toolCache = standardCache(CacheKeys.TOOL_CACHE);
|
||||
expect(MockKeyvRedis).toHaveBeenCalledTimes(1);
|
||||
expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient);
|
||||
expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient, { throwOnErrors: false });
|
||||
expect(toolCache).toBeDefined();
|
||||
});
|
||||
|
||||
|
|
|
|||
15
packages/api/src/cache/cacheFactory.ts
vendored
15
packages/api/src/cache/cacheFactory.ts
vendored
|
|
@ -85,9 +85,17 @@ async function clearRedisNamespace(namespace: string): Promise<void> {
|
|||
* @param namespace - The cache namespace.
|
||||
* @param ttl - Time to live for cache entries.
|
||||
* @param fallbackStore - Optional fallback store if Redis is not used.
|
||||
* @param options.throwOnErrors - Rejects failed Redis operations instead of resolving them as a
|
||||
* miss or a no-op delete, for callers that must know an eviction did not happen. The first
|
||||
* caller's value wins for a shared namespace and TTL.
|
||||
* @returns Cache instance.
|
||||
*/
|
||||
export const standardCache = (namespace: string, ttl?: number, fallbackStore?: object): Keyv => {
|
||||
export const standardCache = (
|
||||
namespace: string,
|
||||
ttl?: number,
|
||||
fallbackStore?: object,
|
||||
options: { throwOnErrors?: boolean } = {},
|
||||
): Keyv => {
|
||||
if (keyvRedisClient && !cacheConfig.FORCED_IN_MEMORY_CACHE_NAMESPACES?.includes(namespace)) {
|
||||
const byTtl = redisCacheMap.get(namespace);
|
||||
const existing = byTtl?.get(ttl);
|
||||
|
|
@ -95,8 +103,9 @@ export const standardCache = (namespace: string, ttl?: number, fallbackStore?: o
|
|||
return existing;
|
||||
}
|
||||
try {
|
||||
const keyvRedis = new KeyvRedis(keyvRedisClient);
|
||||
const cache = new Keyv(keyvRedis, { namespace, ttl });
|
||||
const { throwOnErrors = false } = options;
|
||||
const keyvRedis = new KeyvRedis(keyvRedisClient, { throwOnErrors });
|
||||
const cache = new Keyv(keyvRedis, { namespace, ttl, throwOnErrors });
|
||||
keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX;
|
||||
keyvRedis.keyPrefixSeparator = cacheConfig.GLOBAL_PREFIX_SEPARATOR;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
import { createPinnedOrderHandlers } from './pinned';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
...jest.requireActual('@librechat/data-schemas'),
|
||||
logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -4144,6 +4144,9 @@ export enum CacheKeys {
|
|||
|
||||
export const AUTH_USER_DOC_BY_ID_PREFIX = 'auth-user-doc-byid';
|
||||
|
||||
/** Lifetime of a cached auth user document, and so the longest a stale one can be served. */
|
||||
export const AUTH_USER_DOC_CACHE_TTL_MS = 5000;
|
||||
|
||||
/**
|
||||
* Enum for violation types, used to identify, log, and cache violations.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
|
||||
import {
|
||||
CacheKeys,
|
||||
AUTH_USER_DOC_BY_ID_PREFIX,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
} from 'librechat-data-provider';
|
||||
import type * as t from '~/types';
|
||||
import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';
|
||||
import balanceSchema from '~/schema/balance';
|
||||
|
|
@ -480,6 +484,119 @@ describe('User Methods - Database Tests', () => {
|
|||
expect(cache.delete).toHaveBeenCalledWith(indexKey);
|
||||
});
|
||||
|
||||
describe('when auth cache eviction fails', () => {
|
||||
const cachedKeys = ['auth-cache-key-a', 'auth-cache-key-b'];
|
||||
|
||||
/** Rejects the named operations the way the throwing Redis-backed auth cache does. */
|
||||
function makeFailingCache(failing: { indexRead?: boolean; deleteKey?: string }) {
|
||||
return {
|
||||
get: jest.fn(async () => {
|
||||
if (failing.indexRead) {
|
||||
throw new Error('WRONGTYPE');
|
||||
}
|
||||
return cachedKeys;
|
||||
}),
|
||||
set: jest.fn().mockResolvedValue(true),
|
||||
delete: jest.fn(async (key: string) => {
|
||||
if (key === failing.deleteKey) {
|
||||
throw new Error('redis unavailable');
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function createCachedUser() {
|
||||
enableAuthUserDocCache();
|
||||
const user = await User.create({
|
||||
name: 'Credential User',
|
||||
email: 'credential@example.com',
|
||||
provider: 'openid',
|
||||
});
|
||||
return user._id?.toString() ?? '';
|
||||
}
|
||||
|
||||
test.each<[string, { indexRead?: boolean; deleteKey?: string }]>([
|
||||
['the reverse index read fails', { indexRead: true }],
|
||||
['an indexed document delete fails', { deleteKey: 'auth-cache-key-a' }],
|
||||
])('waits out the cache TTL at the credential barrier when %s', async (_label, failing) => {
|
||||
const userId = await createCachedUser();
|
||||
const cache = makeFailingCache(failing);
|
||||
const delay = jest.fn().mockResolvedValue(undefined);
|
||||
const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });
|
||||
|
||||
const updated = await methodsWithCache.updateUser(userId, {
|
||||
password: 'new-password-hash',
|
||||
credentialsChangedAt: new Date(),
|
||||
});
|
||||
expect(updated?.credentialsChangedAt).toBeInstanceOf(Date);
|
||||
expect(delay).not.toHaveBeenCalled();
|
||||
|
||||
await methodsWithCache.awaitAuthUserDocEviction(userId);
|
||||
|
||||
expect(delay).toHaveBeenCalledTimes(1);
|
||||
expect(delay.mock.calls[0][0]).toBeGreaterThan(AUTH_USER_DOC_CACHE_TTL_MS);
|
||||
expect(cache.delete).not.toHaveBeenCalledWith(`${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`);
|
||||
if (!failing.indexRead) {
|
||||
expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b');
|
||||
}
|
||||
});
|
||||
|
||||
test('passes the barrier at once when only the index delete fails', async () => {
|
||||
const userId = await createCachedUser();
|
||||
const cache = makeFailingCache({ deleteKey: `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}` });
|
||||
const delay = jest.fn().mockResolvedValue(undefined);
|
||||
const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });
|
||||
|
||||
await methodsWithCache.awaitAuthUserDocEviction(userId);
|
||||
|
||||
expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a');
|
||||
expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b');
|
||||
expect(delay).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('passes the barrier at once when a retried eviction succeeds', async () => {
|
||||
const userId = await createCachedUser();
|
||||
const cache = makeFailingCache({ indexRead: true });
|
||||
const delay = jest.fn().mockResolvedValue(undefined);
|
||||
const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });
|
||||
|
||||
await methodsWithCache.updateUser(userId, { credentialsChangedAt: new Date() });
|
||||
cache.get.mockResolvedValue(cachedKeys);
|
||||
await methodsWithCache.awaitAuthUserDocEviction(userId);
|
||||
|
||||
expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a');
|
||||
expect(delay).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('waits at the barrier when the cache can read but not delete', async () => {
|
||||
const userId = await createCachedUser();
|
||||
const delay = jest.fn().mockResolvedValue(undefined);
|
||||
const methodsWithCache = createUserMethods(mongoose, {
|
||||
getCache: () => ({ get: jest.fn(), set: jest.fn() }),
|
||||
delay,
|
||||
});
|
||||
|
||||
await methodsWithCache.awaitAuthUserDocEviction(userId);
|
||||
|
||||
expect(delay).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('keeps updates themselves best effort', async () => {
|
||||
const userId = await createCachedUser();
|
||||
const delay = jest.fn().mockResolvedValue(undefined);
|
||||
const methodsWithCache = createUserMethods(mongoose, {
|
||||
getCache: () => makeFailingCache({ indexRead: true }),
|
||||
delay,
|
||||
});
|
||||
|
||||
const updated = await methodsWithCache.updateUser(userId, { name: 'Renamed' });
|
||||
|
||||
expect(updated?.name).toBe('Renamed');
|
||||
expect(delay).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
test('should invalidate cached auth user documents on delete', async () => {
|
||||
enableAuthUserDocCache();
|
||||
const user = await User.create({
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import mongoose, { FilterQuery } from 'mongoose';
|
||||
import {
|
||||
AUTH_USER_DOC_BY_ID_PREFIX,
|
||||
AUTH_USER_DOC_CACHE_TTL_MS,
|
||||
CacheKeys,
|
||||
type RefillIntervalUnit,
|
||||
type StatefulCodeEnvironment,
|
||||
} from 'librechat-data-provider';
|
||||
import type { IUser, BalanceConfig, CreateUserRequest, UserDeleteResult } from '~/types';
|
||||
import type { CacheStore } from '~/types';
|
||||
import { evictAuthUserDocs } from '~/utils/eviction';
|
||||
import { escapeRegExp } from '~/utils/string';
|
||||
import { signPayload } from '~/crypto';
|
||||
import logger from '~/config/winston';
|
||||
|
||||
/** Default JWT session expiry: 15 minutes in milliseconds */
|
||||
export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15;
|
||||
|
|
@ -19,8 +21,15 @@ const MAX_SUBAGENT_ADMISSION_FENCES = 32;
|
|||
|
||||
interface UserMethodDeps {
|
||||
getCache?: (key: string) => CacheStore | undefined;
|
||||
/** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */
|
||||
delay?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
const wait = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** Timers can fire a millisecond early, so the wait outlasts the cache TTL by this much. */
|
||||
const AUTH_USER_DOC_EXPIRY_MARGIN_MS = 100;
|
||||
|
||||
function isAuthUserDocCacheEnabled(): boolean {
|
||||
return process.env.AUTH_USER_CACHE_MODE === 'on';
|
||||
}
|
||||
|
|
@ -55,6 +64,7 @@ export function createUserMethods(
|
|||
expectedState?: FilterQuery<IUser>,
|
||||
options?: { preserveExpiresAt?: boolean },
|
||||
) => Promise<IUser | null>;
|
||||
awaitAuthUserDocEviction: (userId: string) => Promise<void>;
|
||||
claimSamlIdentity: (
|
||||
userId: string,
|
||||
samlId: string,
|
||||
|
|
@ -333,6 +343,24 @@ export function createUserMethods(
|
|||
return updated;
|
||||
}
|
||||
|
||||
/**
|
||||
* The barrier a credential change passes before it is confirmed. A cached document without
|
||||
* the new credentialsChangedAt keeps pre-change access tokens verifying until it expires, so
|
||||
* eviction is retried and, when it still cannot remove every document, this resolves only
|
||||
* after the cache TTL. Callers revoke sessions and passkeys first: during the wait those
|
||||
* could otherwise mint access tokens issued after the stamp.
|
||||
*/
|
||||
async function awaitAuthUserDocEviction(userId: string): Promise<void> {
|
||||
if (await invalidateAuthUserDocCache(userId)) {
|
||||
return;
|
||||
}
|
||||
logger.warn(
|
||||
'[awaitAuthUserDocEviction] Cached auth documents were not evicted after a credential change; waiting for them to expire',
|
||||
{ userId, waitMs: AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS },
|
||||
);
|
||||
await (deps.delay ?? wait)(AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS);
|
||||
}
|
||||
|
||||
/** Atomically updates a SAML user only when the incoming identity can claim the document. */
|
||||
async function claimSamlIdentity(
|
||||
userId: string,
|
||||
|
|
@ -358,26 +386,20 @@ export function createUserMethods(
|
|||
return updated;
|
||||
}
|
||||
|
||||
async function invalidateAuthUserDocCache(userId: string): Promise<void> {
|
||||
/** Resolves false only when a cached document for the user may still be served. */
|
||||
async function invalidateAuthUserDocCache(userId: string): Promise<boolean> {
|
||||
if (!isAuthUserDocCacheEnabled()) {
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
const cache = deps.getCache?.(CacheKeys.AUTH_USER_DOC);
|
||||
if (!cache?.get || !cache?.delete) {
|
||||
return;
|
||||
if (!cache?.get) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`;
|
||||
const cachedKeys = await cache.get(indexKey);
|
||||
if (Array.isArray(cachedKeys)) {
|
||||
await Promise.all(
|
||||
cachedKeys.map((key) => (typeof key === 'string' ? cache.delete?.(key) : undefined)),
|
||||
);
|
||||
}
|
||||
await cache.delete(indexKey);
|
||||
} catch {
|
||||
// Cache invalidation must not make a user update fail.
|
||||
const remove = cache.delete?.bind(cache);
|
||||
if (!remove) {
|
||||
return false;
|
||||
}
|
||||
return evictAuthUserDocs({ get: (key) => cache.get(key), delete: remove }, { userId });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -863,6 +885,7 @@ export function createUserMethods(
|
|||
countUsers,
|
||||
createUser,
|
||||
updateUser,
|
||||
awaitAuthUserDocEviction,
|
||||
claimSamlIdentity,
|
||||
acceptTerms,
|
||||
searchUsers,
|
||||
|
|
|
|||
111
packages/data-schemas/src/utils/eviction.spec.ts
Normal file
111
packages/data-schemas/src/utils/eviction.spec.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider';
|
||||
import { evictAuthUserDocs } from './eviction';
|
||||
|
||||
const INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-1`;
|
||||
const OTHER_INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-2`;
|
||||
|
||||
/** A Map-backed store that rejects the operations named for a key, as a throwing Redis store does. */
|
||||
function makeStore(failures: { get?: string[]; delete?: string[] } = {}) {
|
||||
const values = new Map<string, unknown>([
|
||||
[INDEX_KEY, ['doc-a', 'doc-b']],
|
||||
['doc-a', { id: 'user-1' }],
|
||||
['doc-b', { id: 'user-1' }],
|
||||
['doc-c', { id: 'user-1' }],
|
||||
[OTHER_INDEX_KEY, ['doc-other']],
|
||||
['doc-other', { id: 'user-2' }],
|
||||
]);
|
||||
return {
|
||||
values,
|
||||
get: jest.fn(async (key: string) => {
|
||||
if (failures.get?.includes(key)) {
|
||||
throw new Error(`GET ${key} failed`);
|
||||
}
|
||||
return values.get(key);
|
||||
}),
|
||||
delete: jest.fn(async (key: string) => {
|
||||
if (failures.delete?.includes(key)) {
|
||||
throw new Error(`DEL ${key} failed`);
|
||||
}
|
||||
return values.delete(key);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function expectOtherUserCached(store: ReturnType<typeof makeStore>) {
|
||||
expect(store.values.get(OTHER_INDEX_KEY)).toEqual(['doc-other']);
|
||||
expect(store.values.get('doc-other')).toEqual({ id: 'user-2' });
|
||||
}
|
||||
|
||||
describe('evictAuthUserDocs', () => {
|
||||
it('lets a retry find a document an earlier failed delete left behind', async () => {
|
||||
const store = makeStore({ delete: ['doc-a'] });
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);
|
||||
|
||||
store.delete.mockImplementation(async (key: string) => store.values.delete(key));
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true);
|
||||
|
||||
expect(store.values.has('doc-a')).toBe(false);
|
||||
expect(store.values.has(INDEX_KEY)).toBe(false);
|
||||
});
|
||||
|
||||
it('deletes the indexed and explicit documents and the index', async () => {
|
||||
const store = makeStore();
|
||||
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe(
|
||||
true,
|
||||
);
|
||||
|
||||
for (const key of [INDEX_KEY, 'doc-a', 'doc-b', 'doc-c']) {
|
||||
expect(store.values.has(key)).toBe(false);
|
||||
}
|
||||
expectOtherUserCached(store);
|
||||
});
|
||||
|
||||
it('reports a failed index read and keeps the index for a retry', async () => {
|
||||
const store = makeStore({ get: [INDEX_KEY] });
|
||||
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
expect(store.values.has('doc-c')).toBe(false);
|
||||
expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);
|
||||
expect(store.values.has('doc-a')).toBe(true);
|
||||
expectOtherUserCached(store);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a string', 'doc-a'],
|
||||
['a list with a non-string entry', ['doc-a', 42]],
|
||||
])('treats a reverse index holding %s as unreadable', async (_label, indexed) => {
|
||||
const store = makeStore();
|
||||
store.values.set(INDEX_KEY, indexed);
|
||||
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);
|
||||
|
||||
expect(store.values.get(INDEX_KEY)).toEqual(indexed);
|
||||
expect(store.values.has('doc-a')).toBe(true);
|
||||
});
|
||||
|
||||
it('reports a failed document delete, deletes the others and keeps the index for a retry', async () => {
|
||||
const store = makeStore({ delete: ['doc-a'] });
|
||||
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);
|
||||
|
||||
expect(store.values.has('doc-a')).toBe(true);
|
||||
expect(store.values.has('doc-b')).toBe(false);
|
||||
expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);
|
||||
expectOtherUserCached(store);
|
||||
});
|
||||
|
||||
it('counts documents as evicted when only the index delete fails', async () => {
|
||||
const store = makeStore({ delete: [INDEX_KEY] });
|
||||
|
||||
await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true);
|
||||
|
||||
expect(store.values.has('doc-a')).toBe(false);
|
||||
expect(store.values.has('doc-b')).toBe(false);
|
||||
expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);
|
||||
expectOtherUserCached(store);
|
||||
});
|
||||
});
|
||||
77
packages/data-schemas/src/utils/eviction.ts
Normal file
77
packages/data-schemas/src/utils/eviction.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider';
|
||||
import logger from '~/config/winston';
|
||||
|
||||
/** Store operations eviction needs; a failure is only seen when the store throws it. */
|
||||
export interface AuthUserDocEvictionStore {
|
||||
get: (key: string) => Promise<unknown>;
|
||||
delete: (key: string) => Promise<unknown>;
|
||||
}
|
||||
|
||||
function isKeyList(value: unknown): value is string[] {
|
||||
return Array.isArray(value) && value.every((key) => typeof key === 'string');
|
||||
}
|
||||
|
||||
function describeError(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the cached auth user documents for a user, attempting every delete even when
|
||||
* another fails. Resolves false when a document may still be served: the reverse index could
|
||||
* not be read, so its keys are unknown, or a document delete failed. The index is deleted only
|
||||
* after every document it names is gone, so a retry can still find what an earlier attempt
|
||||
* left behind. A failed delete of the index alone leaves only the names of deleted keys, so it
|
||||
* is logged and still counts as evicted. Never throws, so callers keep their own write results.
|
||||
*/
|
||||
export async function evictAuthUserDocs(
|
||||
store: AuthUserDocEvictionStore,
|
||||
input: { userId?: string; cacheKey?: string },
|
||||
): Promise<boolean> {
|
||||
const keys = new Set<string>(input.cacheKey ? [input.cacheKey] : []);
|
||||
const indexKey = input.userId ? `${AUTH_USER_DOC_BY_ID_PREFIX}:${input.userId}` : undefined;
|
||||
let indexRead = true;
|
||||
if (indexKey) {
|
||||
try {
|
||||
const indexed = await store.get(indexKey);
|
||||
if (indexed != null && !isKeyList(indexed)) {
|
||||
throw new Error('Reverse index is not a list of cache keys');
|
||||
}
|
||||
for (const key of indexed ?? []) {
|
||||
keys.add(key);
|
||||
}
|
||||
} catch (error) {
|
||||
indexRead = false;
|
||||
logger.warn('[authUserDocCache] Reverse index read failed during eviction', {
|
||||
userId: input.userId,
|
||||
error: describeError(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled([...keys].map((key) => store.delete(key)));
|
||||
let documentsDeleted = true;
|
||||
for (const result of results) {
|
||||
if (result.status === 'rejected') {
|
||||
documentsDeleted = false;
|
||||
logger.warn('[authUserDocCache] Cached document delete failed during eviction', {
|
||||
userId: input.userId,
|
||||
error: describeError(result.reason),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!indexRead || !documentsDeleted) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (indexKey) {
|
||||
try {
|
||||
await store.delete(indexKey);
|
||||
} catch (error) {
|
||||
logger.warn('[authUserDocCache] Reverse index delete failed during eviction', {
|
||||
userId: input.userId,
|
||||
error: describeError(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -8,5 +8,6 @@ export * from './objectId';
|
|||
export * from './yaml';
|
||||
export * from './stripUIResourceMarkers';
|
||||
export * from './fading';
|
||||
export * from './eviction';
|
||||
export { buildIndexWithRetry, createIndexesWithRetry, isIndexBuildInProgress } from './retry';
|
||||
export type { IndexBuildOptions } from './retry';
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue