🗄️ fix: Restore Transaction Capability Detection (#17735)

* fix: restore transaction capability detection

* fix: keep authorization and response reads on the primary
This commit is contained in:
Dustin Healy 2026-11-10 21:04:07 -08:00 • committed by GitHub
parent f70aff8420
commit b83bcdc572
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 503 additions and 26 deletions

View file

@ -0,0 +1,166 @@
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import { MongoMemoryServer, MongoMemoryReplSet } from 'mongodb-memory-server';
import { ResourceType, AccessRoleIds, PrincipalType } from 'librechat-data-provider';
import {
createModels,
createMethods,
supportsTransactions,
tenantStorage,
RoleBits,
} from '@librechat/data-schemas';
import { AccessControlService } from './accessControlService';
const backends = ['standalone', 'replica', ...(process.env.FERRETDB_URI ? ['ferret'] : [])];
describe.each(backends)('ACL updates on %s with secondaryPreferred defaults', (backend) => {
const db = new mongoose.Mongoose();
let server: MongoMemoryServer | MongoMemoryReplSet | undefined;
let service: AccessControlService;
const roleReads: Array<{ address: string; inTransaction: boolean }> = [];
const resourceId = new mongoose.Types.ObjectId();
const userId = new mongoose.Types.ObjectId();
const grantedBy = new mongoose.Types.ObjectId();
const request = {
resourceType: ResourceType.AGENT,
resourceId,
grantedBy,
updatedPrincipals: [
{
type: PrincipalType.USER,
id: userId.toString(),
accessRoleId: AccessRoleIds.AGENT_VIEWER,
},
],
};
beforeAll(async () => {
if (backend !== 'ferret') {
server =
backend === 'replica'
? await MongoMemoryReplSet.create({ replSet: { count: 2 } })
: await MongoMemoryServer.create();
}
await db.connect(server?.getUri() ?? process.env.FERRETDB_URI!, {
dbName: `acl_transactions_${randomUUID().replaceAll('-', '')}`,
readPreference: 'secondaryPreferred',
autoIndex: false,
monitorCommands: true,
});
createModels(db);
await Promise.all(
['AclEntry', 'AccessRole', 'Role', 'ResourceLock'].map((name) =>
db.model(name).createIndexes(),
),
);
await createMethods(db).seedDefaultRoles();
service = new AccessControlService(db);
db.connection.getClient().on('commandStarted', (event) => {
if (
event.commandName === 'find' &&
event.command.find === db.model('AccessRole').collection.name
) {
roleReads.push({
address: event.address,
inTransaction: event.command.autocommit === false,
});
}
});
}, 60000);
beforeEach(async () => {
await db.model('AclEntry').deleteMany({});
roleReads.length = 0;
});
afterAll(async () => {
if (db.connection.readyState === 1) await db.connection.dropDatabase();
await db.disconnect();
await server?.stop();
});
it('detects the real capability and commits a permission update', async () => {
await expect(supportsTransactions(db)).resolves.toBe(backend === 'replica');
await service.bulkUpdateResourcePermissions(request);
expect(
await db
.model('AclEntry')
.countDocuments({ resourceId, principalId: userId })
.read('primary'),
).toBe(1);
if (backend === 'replica') {
const hello = await db.connection
.db!.admin()
.command({ hello: 1 }, { readPreference: 'primary' });
expect(roleReads).toEqual([{ address: hello.primary, inTransaction: true }]);
}
});
it('keeps the existing permission when the audited write is refused', async () => {
await service.bulkUpdateResourcePermissions(request);
await expect(
service.bulkUpdateResourcePermissions({
resourceType: ResourceType.AGENT,
resourceId,
grantedBy,
revokedPrincipals: [{ type: PrincipalType.USER, id: userId.toString() }],
beforeWrite: async () => {
throw new Error('audit unavailable');
},
}),
).rejects.toThrow('audit unavailable');
expect(
await db
.model('AclEntry')
.countDocuments({ resourceId, principalId: userId })
.read('primary'),
).toBe(1);
});
if (backend === 'replica') {
it('leaves a caller-owned transaction open for rollback', async () => {
const session = await db.startSession();
session.startTransaction({ readPreference: 'primary' });
try {
await service.bulkUpdateResourcePermissions({ ...request, session });
expect(session.inTransaction()).toBe(true);
expect(await db.model('AclEntry').countDocuments({ resourceId }).read('primary')).toBe(0);
await session.abortTransaction();
expect(await db.model('AclEntry').countDocuments({ resourceId }).read('primary')).toBe(0);
} finally {
await session.endSession();
}
});
it('resolves a tenant role override inside the caller-owned transaction', async () => {
const tenantId = 'role-transaction';
const session = await db.startSession();
session.startTransaction({ readPreference: 'primary' });
try {
await tenantStorage.run({ tenantId }, async () => {
await db.model('AccessRole').create(
[
{
accessRoleId: AccessRoleIds.AGENT_VIEWER,
name: 'Tenant viewer override',
resourceType: ResourceType.AGENT,
permBits: RoleBits.EDITOR,
tenantId,
},
],
{ session },
);
await service.bulkUpdateResourcePermissions({ ...request, session });
const entry = await db
.model('AclEntry')
.findOne({ resourceId, principalId: userId })
.session(session);
expect(entry?.permBits).toBe(RoleBits.EDITOR);
expect(session.inTransaction()).toBe(true);
});
} finally {
await session.abortTransaction();
await session.endSession();
}
});
}
});

View file

@ -573,7 +573,7 @@ export class AccessControlService {
}
if (!localSession && supportsTransactions) {
localSession = await this._mongoose.startSession();
localSession.startTransaction();
localSession.startTransaction({ readPreference: 'primary' });
shouldEndSession = true;
}
const sessionOptions = localSession ? { session: localSession } : {};
@ -607,8 +607,8 @@ export class AccessControlService {
/** An invalid id is reported per principal below. */
}
}
const [roles, currentEntries] = await Promise.all([
this._dbMethods.findRolesByResourceType(resourceType),
const loadRoles = () => this._dbMethods.findRolesByResourceType(resourceType, localSession);
const loadEntries = () =>
affectedPrincipals.length > 0
? this._dbMethods.findEntriesByPrincipalsAndResource(
affectedPrincipals,
@ -617,8 +617,11 @@ export class AccessControlService {
localSession,
true,
)
: Promise.resolve([]),
]);
: Promise.resolve([]);
// A transaction cannot run parallel operations on its session.
const [roles, currentEntries] = localSession?.inTransaction()
? ([await loadRoles(), await loadEntries()] as const)
: await Promise.all([loadRoles(), loadEntries()]);
const rolesMap = new Map(roles.map((role) => [role.accessRoleId, role]));
const results: BulkPermissionUpdateResult = {
granted: [],

View file

@ -1,5 +1,5 @@
import { AccessRoleIds, ResourceType } from 'librechat-data-provider';
import type { Model, Types, DeleteResult } from 'mongoose';
import type { Model, Types, ClientSession, DeleteResult } from 'mongoose';
import type { IAccessRole } from '~/types';
import { getTenantId, runAsSystem, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { RoleBits } from '~/common';
@ -12,8 +12,14 @@ export function createAccessRoleMethods(mongoose: typeof import('mongoose')): {
) => Promise<IAccessRole | null>;
deleteRole: (accessRoleId: string | Types.ObjectId) => Promise<DeleteResult>;
seedDefaultRoles: () => Promise<Record<string, IAccessRole>>;
findRoleByIdentifier: (accessRoleId: string | Types.ObjectId) => Promise<IAccessRole | null>;
findRolesByResourceType: (resourceType: string) => Promise<IAccessRole[]>;
findRoleByIdentifier: (
accessRoleId: string | Types.ObjectId,
session?: ClientSession,
) => Promise<IAccessRole | null>;
findRolesByResourceType: (
resourceType: string,
session?: ClientSession,
) => Promise<IAccessRole[]>;
} {
/** True inside a tenant's own context; false with no context or a system one. */
function inTenantScope(): boolean {
@ -41,7 +47,7 @@ export function createAccessRoleMethods(mongoose: typeof import('mongoose')): {
*
* The unique index on `(accessRoleId, tenantId)` makes the substitution unambiguous.
*/
async function visibleTenantRoles(): Promise<IAccessRole[]> {
async function visibleTenantRoles(session?: ClientSession): Promise<IAccessRole[]> {
const tenantId = getTenantId();
const AccessRole = mongoose.models.AccessRole as Model<IAccessRole>;
@ -49,6 +55,8 @@ export function createAccessRoleMethods(mongoose: typeof import('mongoose')): {
* query inside it. MongoDB's null match includes absent tenantId fields. */
const roles = await runAsSystem(async () =>
AccessRole.find({ tenantId: { $in: [tenantId, null] } })
.session(session ?? null)
.read('primary')
.lean<IAccessRole[]>()
.exec(),
);
@ -70,12 +78,17 @@ export function createAccessRoleMethods(mongoose: typeof import('mongoose')): {
*/
async function findRoleByIdentifier(
accessRoleId: string | Types.ObjectId,
session?: ClientSession,
): Promise<IAccessRole | null> {
const AccessRole = mongoose.models.AccessRole as Model<IAccessRole>;
if (!inTenantScope()) {
return await AccessRole.findOne({ accessRoleId }).lean<IAccessRole>().exec();
return await AccessRole.findOne({ accessRoleId })
.session(session ?? null)
.read('primary')
.lean<IAccessRole>()
.exec();
}
const roles = await visibleTenantRoles();
const roles = await visibleTenantRoles(session);
return roles.find((role) => String(role.accessRoleId) === String(accessRoleId)) ?? null;
}
@ -88,16 +101,23 @@ export function createAccessRoleMethods(mongoose: typeof import('mongoose')): {
* @param resourceType - The type of resource ('agent', 'project', 'file')
* @returns Array of role documents
*/
async function findRolesByResourceType(resourceType: string): Promise<IAccessRole[]> {
async function findRolesByResourceType(
resourceType: string,
session?: ClientSession,
): Promise<IAccessRole[]> {
const AccessRole = mongoose.models.AccessRole as Model<IAccessRole>;
if (!inTenantScope()) {
return await AccessRole.find({ resourceType }).lean<IAccessRole[]>().exec();
return await AccessRole.find({ resourceType })
.session(session ?? null)
.read('primary')
.lean<IAccessRole[]>()
.exec();
}
/** Filtered after the merge, so a tenant copy that moves an identifier to another
* resource type withdraws the global row from this list instead of leaving it
* visible — and callers such as `bulkUpdatePermissions`, which key this result by
* `accessRoleId`, cannot grant bits the tenant has superseded. */
const roles = await visibleTenantRoles();
const roles = await visibleTenantRoles(session);
return roles.filter((role) => role.resourceType === resourceType);
}

View file

@ -346,7 +346,9 @@ export function createResponseStorage(
if (!current) return conflict;
const keys = responseScopeKeys(owner, current.conversationId);
await initializeResponseScopes(mongoose, keys);
const session = await mongoose.startSession();
const session = await mongoose.startSession({
defaultTransactionOptions: { readPreference: 'primary' },
});
try {
let result = conflict;
await session.withTransaction(async () => {
@ -757,7 +759,9 @@ export function createResponseStorage(
const prior = await model.findOne(filter).select('conversationId').lean();
const keys = responseScopeKeys(input, prior?.conversationId ?? input.conversationId);
await initializeResponseScopes(mongoose, keys);
const session = await mongoose.startSession();
const session = await mongoose.startSession({
defaultTransactionOptions: { readPreference: 'primary' },
});
try {
let result: ResponseReservationResult | undefined;
await session.withTransaction(async () => {

View file

@ -0,0 +1,152 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createResponseRetention,
createResponseConversationReplacement,
closeResponseScopes,
} from './responseScope';
import { tenantStorage } from '~/config/tenantContext';
import { createResponseStorage } from './response';
import { createModels } from '~/models';
describe('Response transaction owners with secondaryPreferred defaults', () => {
const db = new mongoose.Mongoose();
const owner = { userId: 'owner', tenantId: 'tenant' };
const scoped = <T>(work: () => Promise<T>) =>
tenantStorage.run({ tenantId: owner.tenantId }, work);
let mongo: MongoMemoryReplSet;
let secondary: mongoose.mongo.MongoClient;
async function withReplicationPaused<T>(work: () => Promise<T>): Promise<T> {
await db.connection
.db!.collection('replication_barrier')
.insertOne({}, { writeConcern: { w: 'majority' } });
const admin = secondary.db('admin');
const configured = await admin.command({
configureFailPoint: 'stopReplProducer',
mode: 'alwaysOn',
});
try {
await admin.command({
waitForFailPoint: 'stopReplProducer',
timesEntered: configured.count + 1,
maxTimeMS: 10000,
});
return await work();
} finally {
await admin.command({ configureFailPoint: 'stopReplProducer', mode: 'off' });
}
}
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 2 },
instanceOpts: [
{ args: ['--setParameter', 'enableTestCommands=1'] },
{ args: ['--setParameter', 'enableTestCommands=1'] },
],
});
await db.connect(mongo.getUri(), {
readPreference: 'secondaryPreferred',
autoIndex: false,
writeConcern: { w: 1 },
});
createModels(db);
await Promise.all(
[
'StoredResponse',
'ResponseScope',
'ChannelReply',
'ChannelFeedback',
'ChannelFeedbackEvent',
].map((name) => db.model(name).createCollection()),
);
await db.model('StoredResponse').createIndexes();
const hello = await db.connection
.db!.admin()
.command({ hello: 1 }, { readPreference: 'primary' });
const address = (hello.hosts as string[]).find((host) => host !== hello.primary);
if (!address) throw new Error('Missing secondary fixture');
secondary = new mongoose.mongo.MongoClient(`mongodb://${address}`, {
directConnection: true,
readPreference: 'secondary',
});
await secondary.connect();
}, 60000);
afterAll(async () => {
await secondary?.close();
await db.disconnect();
await mongo?.stop();
});
it('reserves, mutates and retains a response through its real owned transactions', async () =>
scoped(() =>
withReplicationPaused(async () => {
const storage = createResponseStorage(db);
const result = await storage.reserve({
...owner,
dispatch: true,
responseId: 'resp_reader_default',
requestKey: 'reader-default',
fingerprint: 'same-request',
agentId: 'agent',
requestJSON: JSON.stringify({ model: 'agent', input: 'hello', background: true }),
conversationId: 'conversation',
parentMessageId: 'root',
});
expect(result.ok).toBe(true);
expect(
await secondary
.db(db.connection.name)
.collection(db.model('StoredResponse').collection.name)
.findOne({ responseId: 'resp_reader_default' }),
).toBeNull();
expect(
await storage.settleDispatch(owner, 'resp_reader_default', {
kind: 'create',
id: 'resp_reader_default',
}),
).toBe(true);
expect(
(await storage.requestCancellation(owner, 'resp_reader_default', 123))
?.cancellationRequestedAt,
).toBe(123);
const expiry = new Date(Date.now() + 60000);
await createResponseRetention(db)(owner, 'conversation', expiry);
expect((await storage.read(owner, 'resp_reader_default'))?.expiredAt).toEqual(expiry);
expect(await createResponseConversationReplacement(db)(owner, 'conversation')).toBe(false);
expect(await createResponseConversationReplacement(db)(owner, 'unused-conversation')).toBe(
true,
);
}),
));
it('observes closure even while the secondary still exposes the live response', async () =>
scoped(async () => {
const storage = createResponseStorage(db);
await storage.reserve({
...owner,
responseId: 'resp_closure',
requestKey: 'closure',
fingerprint: 'closure',
agentId: 'agent',
requestJSON: '{}',
conversationId: 'closed-conversation',
parentMessageId: 'root',
});
await withReplicationPaused(async () => {
await closeResponseScopes(db, owner, ['closed-conversation']);
expect(
await secondary
.db(db.connection.name)
.collection(db.model('StoredResponse').collection.name)
.findOne({ responseId: 'resp_closure' }),
).not.toBeNull();
expect(await storage.read(owner, 'resp_closure')).toBeNull();
expect((await storage.readCleanupIdentity(owner, 'resp_closure'))?.retirement).toBe(
'conversation_deleted',
);
});
}));
});

View file

@ -29,7 +29,9 @@ export function createResponseConversationReplacement(
}
const key = responseScopeKeys(owner, conversationId)[1];
await initializeResponseScopes(mongoose, [key]);
const session = await mongoose.startSession();
const session = await mongoose.startSession({
defaultTransactionOptions: { readPreference: 'primary' },
});
try {
let allowed = false;
await session.withTransaction(async () => {
@ -123,7 +125,9 @@ export function createResponseRetention(
};
if (parentSession) return apply(parentSession);
if (!supported) return apply();
const session = await mongoose.startSession();
const session = await mongoose.startSession({
defaultTransactionOptions: { readPreference: 'primary' },
});
try {
await session.withTransaction(() => apply(session));
} finally {

View file

@ -213,7 +213,8 @@ export function createStoredResponseModel(
required: false,
},
},
{ timestamps: true, minimize: false },
// This journal drives admission and recovery; stale replica reads are not authoritative.
{ timestamps: true, minimize: false, read: 'primary' },
);
schema.index({ tenantId: 1, userId: 1, responseId: 1 }, { unique: true });
schema.index({ expiredAt: 1 }, { expireAfterSeconds: 0 });

View file

@ -20,7 +20,8 @@ export function createResponseScopeModel(
revision: { type: Number, required: true, default: 0 },
expiredAt: Date,
},
{ versionKey: false },
// Closure and retention must be observed before any durable data is disclosed or mutated.
{ versionKey: false, read: 'primary' },
);
return mongoose.model<ResponseScope>('ResponseScope', schema);
}

View file

@ -1,4 +1,8 @@
import { getTransactionSupport } from './transactions';
import mongoose from 'mongoose';
import { MongoMemoryServer, MongoMemoryReplSet } from 'mongodb-memory-server';
import { getTransactionSupport, supportsTransactions } from './transactions';
jest.mock('~/config/winston', () => ({ debug: jest.fn() }));
describe('getTransactionSupport', () => {
// The cache-hit path does not touch the database, so a stub mongoose is fine.
@ -9,3 +13,122 @@ describe('getTransactionSupport', () => {
await expect(getTransactionSupport(mongoose, false)).resolves.toBe(false);
});
});
describe('transaction capability against real MongoDB', () => {
const standalone = new mongoose.Mongoose();
const replica = new mongoose.Mongoose();
let mongo: MongoMemoryServer;
let repl: MongoMemoryReplSet;
beforeAll(async () => {
[mongo, repl] = await Promise.all([
MongoMemoryServer.create(),
MongoMemoryReplSet.create({ replSet: { count: 1 } }),
]);
await Promise.all([
standalone.connect(mongo.getUri()),
replica.connect(repl.getUri(), { readPreference: 'secondaryPreferred' }),
]);
}, 60000);
afterEach(() => jest.restoreAllMocks());
afterAll(async () => {
await Promise.all([standalone.disconnect(), replica.disconnect()]);
await Promise.all([mongo?.stop(), repl?.stop()]);
});
it('returns false for standalone MongoDB without writing probe data', async () => {
await expect(supportsTransactions(standalone)).resolves.toBe(false);
expect(
await standalone.connection.db!.collection('__transaction_test__').countDocuments(),
).toBe(0);
});
it('supports transactions despite a secondaryPreferred client default', async () => {
await expect(supportsTransactions(replica)).resolves.toBe(true);
});
it('shares an inflight probe only within the same connection', async () => {
const replicaSessions = jest.spyOn(replica, 'startSession');
const standaloneSessions = jest.spyOn(standalone, 'startSession');
await expect(
Promise.all([
getTransactionSupport(replica, null),
getTransactionSupport(replica, null),
getTransactionSupport(standalone, null),
]),
).resolves.toEqual([true, true, false]);
expect(replicaSessions).toHaveBeenCalledTimes(1);
expect(standaloneSessions).toHaveBeenCalledTimes(1);
});
it('propagates a failed shared probe and retries after recovery', async () => {
const failure = new mongoose.mongo.MongoNetworkError('probe connection lost');
const sessions = jest.spyOn(replica, 'startSession').mockRejectedValueOnce(failure);
const results = await Promise.allSettled([
getTransactionSupport(replica, null),
getTransactionSupport(replica, null),
]);
expect(results).toEqual([
{ status: 'rejected', reason: failure },
{ status: 'rejected', reason: failure },
]);
expect(sessions).toHaveBeenCalledTimes(1);
await expect(getTransactionSupport(replica, null)).resolves.toBe(true);
expect(sessions).toHaveBeenCalledTimes(2);
});
it.each([
[59, "no such command: 'commitTransaction'", false],
[59, "no such command: 'find'", true],
[20, 'an unrelated illegal operation', true],
[13, 'not authorized', true],
[18, 'authentication failed', true],
])(
'classifies commit failure %i (%s) without hiding operational errors',
async (code, errmsg, throws) => {
const session = await replica.startSession();
jest.spyOn(replica, 'startSession').mockResolvedValueOnce(session);
const end = jest.spyOn(session, 'endSession');
const error = new mongoose.mongo.MongoServerError({ code, errmsg });
jest.spyOn(session, 'commitTransaction').mockRejectedValueOnce(error);
if (throws) await expect(supportsTransactions(replica)).rejects.toBe(error);
else await expect(supportsTransactions(replica)).resolves.toBe(false);
expect(end).toHaveBeenCalledTimes(1);
},
);
it('retries a missing DocumentDB probe namespace after concurrent initialization', async () => {
const session = await replica.startSession();
const starts = jest.spyOn(replica, 'startSession').mockResolvedValueOnce(session);
jest.spyOn(session, 'commitTransaction').mockRejectedValueOnce(
new mongoose.mongo.MongoServerError({
code: 263,
errmsg: 'non-existent collection in transaction',
}),
);
const create = jest
.spyOn(replica.connection.db!, 'createCollection')
.mockRejectedValueOnce(
new mongoose.mongo.MongoServerError({ code: 48, errmsg: 'namespace already exists' }),
);
await expect(supportsTransactions(replica)).resolves.toBe(true);
expect(create).toHaveBeenCalledWith('__transaction_test__');
expect(starts).toHaveBeenCalledTimes(2);
});
it('does not hide operational failure while creating the probe namespace', async () => {
const session = await replica.startSession();
jest.spyOn(replica, 'startSession').mockResolvedValueOnce(session);
jest.spyOn(session, 'commitTransaction').mockRejectedValueOnce(
new mongoose.mongo.MongoServerError({
code: 263,
errmsg: 'non-existent collection in transaction',
}),
);
const error = new mongoose.mongo.MongoServerError({ code: 13, errmsg: 'not authorized' });
jest.spyOn(replica.connection.db!, 'createCollection').mockRejectedValueOnce(error);
await expect(supportsTransactions(replica)).rejects.toBe(error);
});
});

View file

@ -4,17 +4,20 @@ export const CANCEL_RATE = 1.15;
const TRANSACTION_PROBE_COLLECTION = '__transaction_test__';
function unsupportedTransaction(error: unknown, mongoose: typeof import('mongoose')): boolean {
if (!(error instanceof mongoose.mongo.MongoServerError)) return false;
return (
error instanceof mongoose.mongo.MongoServerError &&
error.code === 20 &&
/Transaction numbers are only allowed on a replica set member or mongos/i.test(error.message)
(error.code === 20 &&
/Transaction numbers are only allowed on a replica set member or mongos/i.test(
error.message,
)) ||
(error.code === 59 && /^no such command: ['"]commitTransaction['"]$/i.test(error.message))
);
}
async function probeTransaction(mongoose: typeof import('mongoose')): Promise<boolean> {
const session = await mongoose.startSession();
try {
session.startTransaction();
session.startTransaction({ readPreference: 'primary' });
const db = mongoose.connection.db;
if (!db) throw new Error('Transaction capability probe requires a connected database');
await db.collection(TRANSACTION_PROBE_COLLECTION).findOne({}, { session });
@ -33,7 +36,7 @@ async function probeTransaction(mongoose: typeof import('mongoose')): Promise<bo
}
}
/** Only confirmed standalone incompatibility is false; operational failures stay retryable. */
/** Only confirmed transaction incompatibility is false; operational failures stay retryable. */
export const supportsTransactions = async (
mongoose: typeof import('mongoose'),
): Promise<boolean> => {