diff --git a/api/server/controllers/PermissionsController.js b/api/server/controllers/PermissionsController.js index fabc561093..5188729a06 100644 --- a/api/server/controllers/PermissionsController.js +++ b/api/server/controllers/PermissionsController.js @@ -173,6 +173,7 @@ const updateResourcePermissions = async (req, res) => { const results = await bulkUpdateResourcePermissions({ resourceType, resourceId, + maxWriteAttempts: req.config?.config?.permissions?.maxWriteAttempts, updatedPrincipals: validatedPrincipals, revokedPrincipals, grantedBy: userId, diff --git a/api/server/controllers/__tests__/PermissionsController.spec.js b/api/server/controllers/__tests__/PermissionsController.spec.js index 0b83041a56..17248e61b6 100644 --- a/api/server/controllers/__tests__/PermissionsController.spec.js +++ b/api/server/controllers/__tests__/PermissionsController.spec.js @@ -339,8 +339,12 @@ describe('PermissionsController', () => { }); const res = createMockRes(); + req.config = { config: { permissions: { maxWriteAttempts: 7 } } }; await updateResourcePermissions(req, res); + expect(mockBulkUpdateResourcePermissions).toHaveBeenCalledWith( + expect.objectContaining({ maxWriteAttempts: 7 }), + ); expect(mockRecordAuditEntry).toHaveBeenCalledWith( expect.objectContaining({ action: 'permission.insights_assigned', diff --git a/api/server/services/PermissionService.spec.js b/api/server/services/PermissionService.spec.js index a2dae30366..0ab7411493 100644 --- a/api/server/services/PermissionService.spec.js +++ b/api/server/services/PermissionService.spec.js @@ -1062,9 +1062,9 @@ describe('PermissionService', () => { ); } - const originalBulkWriteAclEntries = db.bulkWriteAclEntries; + const originalBulkWriteAclEntries = AclEntry.bulkWrite.bind(AclEntry); const bulkWriteSpy = jest - .spyOn(db, 'bulkWriteAclEntries') + .spyOn(AclEntry, 'bulkWrite') .mockImplementationOnce(async (...args) => { await AclEntry.updateOne( { @@ -1091,6 +1091,7 @@ describe('PermissionService', () => { ], grantedBy: grantedById, }); + expect(bulkWriteSpy).toHaveBeenCalled(); } finally { bulkWriteSpy.mockRestore(); } @@ -1107,9 +1108,9 @@ describe('PermissionService', () => { ); test('removes a stale-read upsert after a concurrent deletion and audit failure', async () => { - const originalBulkWriteAclEntries = db.bulkWriteAclEntries; + const originalBulkWriteAclEntries = AclEntry.bulkWrite.bind(AclEntry); const bulkWriteSpy = jest - .spyOn(db, 'bulkWriteAclEntries') + .spyOn(AclEntry, 'bulkWrite') .mockImplementationOnce(async (...args) => { await AclEntry.deleteOne({ principalType: PrincipalType.USER, diff --git a/docs/permissions/acl-write-compatibility.md b/docs/permissions/acl-write-compatibility.md new file mode 100644 index 0000000000..89655dd1e1 --- /dev/null +++ b/docs/permissions/acl-write-compatibility.md @@ -0,0 +1,52 @@ +# ACL write compatibility + +Role-only sharing preserves independently administered permission bits while updating role bits, +roleId, grantedBy and grantedAt in one conditional document update. The persistence layer also uses +that same mutation operation for add/remove bit requests. Removing a bit wins if it appears in both +masks. Neither path uses MongoDB bit-update operators. + +## Deployment configuration + +```yaml +permissions: + maxWriteAttempts: 3 +``` + +The sharing endpoint passes this setting to the persistence operation. The value must be an integer +from 1 through 100, including the initial attempt. Internal callers can override the same validated +budget per call. A session alone does not imply a transaction: comparison reads use primary unless +an active transaction owns the read preference. + +## Concurrency and failures + +Each observed ACL document is its own unit of completion. A conditional write checks the snapshot's +bits and role/audit metadata. Only a definite no-match retries that document. Already completed +documents are never replayed because a different document conflicts. Duplicate ACL documents are +processed individually and keep their own independent bits; this change does not deduplicate them +or add a uniqueness constraint. + +If an observed ACL is deleted during mutation, the request fails rather than recreating that grant. +A missing principal at the initial read can still be granted access. If another writer inserts it +first, the operation reads and updates that entry instead of treating a no-op upsert as success. +Concurrent inserts can still create duplicate identities without a unique index; each completed +write is atomic, but the entire resource is not serialized. + +Standalone deployments do not get batch atomicity. If a later document fails, earlier writes can +remain committed. Errors include completedEntries (acknowledged writes before failure) and whether +an active transaction was present. This count does not settle ambiguous network failures or prove a +transaction committed. MongoDB error labels are retained for the transaction owner's handling. +No automatic retry is performed for ambiguous write errors or transaction write-conflict errors. + +## Upgrading + +No data migration, collection reset, or permission reset is required. Missing legacy permBits +fields are interpreted as zero. Nonnegative 31-bit integers preserve bits outside the known enum. +Null, fractional, negative, or larger values fail explicitly instead of being truncated. Investigate +those rows against authoritative role and audit records; do not blanket-reset grants. + +Old binaries do not gain these safeguards during a rolling upgrade. Drain old writers before relying +on the new behavior. Storage remains unchanged, but rolling back also restores the old implementation's +limitations. A separate Insights field and any associated migration are not part of these PRs. + +MongoDB-backed tests cover the update contract. Live DocumentDB and Cosmos DB compatibility has not +been certified by this change; the original reported engine/version restriction remains unverified. diff --git a/librechat.example.yaml b/librechat.example.yaml index 6b96e4b3e1..260912aa78 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1444,3 +1444,7 @@ endpoints: # - id: anthropic_api_key # label: Anthropic API key # regex: "sk-ant-[A-Za-z0-9_-]{20,}" + +# ACL writes retry only conflicts on an individual document. +# permissions: +# maxWriteAttempts: 3 # Integer 1-100, including the first attempt. diff --git a/packages/api/src/acl/accessControlService.ts b/packages/api/src/acl/accessControlService.ts index f82e23578e..0f52e845c8 100644 --- a/packages/api/src/acl/accessControlService.ts +++ b/packages/api/src/acl/accessControlService.ts @@ -3,7 +3,6 @@ import { createMethods, getTransactionSupport, logger, - RoleBits, runAfterTransaction, } from '@librechat/data-schemas'; import { @@ -411,6 +410,7 @@ export class AccessControlService { revokedPrincipals = [], grantedBy, session, + maxWriteAttempts, }: { resourceType: ResourceType; resourceId: string | Types.ObjectId; @@ -418,6 +418,7 @@ export class AccessControlService { revokedPrincipals?: BulkPrincipal[]; grantedBy: string | Types.ObjectId; session?: ClientSession; + maxWriteAttempts?: number; }): Promise { const supportsTransactions = await getTransactionSupport( this._mongoose, @@ -459,6 +460,8 @@ export class AccessControlService { errors: [], }; const bulkWrites: Parameters[0] = []; + /** Role-only edits go through the atomic guarded write, not this batch. */ + const roleBitsWrites: Parameters[0] = []; const insightsChangesByBulkWriteIndex = new Map(); const updatedPrincipalKey = (principal: BulkPrincipal | null | undefined) => { @@ -572,14 +575,6 @@ export class AccessControlService { grantedBy, grantedAt, }, - ...(preserveInsights && { - $bit: { - permBits: { - or: role.permBits & RoleBits.OWNER, - and: ~(RoleBits.OWNER & ~role.permBits), - }, - }, - }), $setOnInsert: { principalType: principal.type, resourceType, @@ -591,9 +586,18 @@ export class AccessControlService { }, }; const bulkWriteIndex = bulkWrites.length; - bulkWrites.push({ - updateMany: { filter: query, update, upsert: true }, - }); + if (preserveInsights) { + roleBitsWrites.push({ + filter: query, + insert: update.$setOnInsert, + roleBits: role.permBits, + metadata: update.$set, + }); + } else { + bulkWrites.push({ + updateMany: { filter: query, update, upsert: true }, + }); + } results.granted.push({ type: principal.type, id: principal.id, @@ -635,6 +639,13 @@ export class AccessControlService { } } + if (roleBitsWrites.length > 0) { + await this._dbMethods.replaceRoleBits(roleBitsWrites, { + ...sessionOptions, + maxAttempts: maxWriteAttempts, + }); + } + if (bulkWrites.length > 0) { const bulkWriteResult = await this._dbMethods.bulkWriteAclEntries( bulkWrites, diff --git a/packages/api/src/acl/rolePermissions.spec.ts b/packages/api/src/acl/rolePermissions.spec.ts new file mode 100644 index 0000000000..74fb297537 --- /dev/null +++ b/packages/api/src/acl/rolePermissions.spec.ts @@ -0,0 +1,604 @@ +import mongoose, { Types } from 'mongoose'; +import { MongoMemoryServer, MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + ResourceType, + AccessRoleIds, + PermissionBits, + PrincipalModel, + PrincipalType, +} from 'librechat-data-provider'; +import { + RoleBits, + createModels, + createMethods, + tenantStorage, + permissionBitSupersets, +} from '@librechat/data-schemas'; +import type { IAclEntry } from '@librechat/data-schemas'; +import type { Model } from 'mongoose'; +import { AccessControlService } from './accessControlService'; + +let mongo: MongoMemoryServer; +let entries: Model; +let methods: ReturnType; +let service: AccessControlService; +const resourceId = new Types.ObjectId(); +const userId = new Types.ObjectId(); +const grantedBy = new Types.ObjectId(); +const filter = { + principalType: PrincipalType.USER, + principalId: userId, + resourceType: ResourceType.AGENT, + resourceId, +}; + +beforeAll(async () => { + mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } }); + await mongoose.connect(mongo.getUri()); + createModels(mongoose); + entries = mongoose.models.AclEntry as Model; + methods = createMethods(mongoose); + await methods.seedDefaultRoles(); +}); + +beforeEach(async () => { + await entries.deleteMany({}); + service = new AccessControlService(mongoose, methods); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +async function seed(permBits: number, tenantId?: string): Promise { + const role = await methods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER); + await entries.create({ + ...filter, + principalModel: PrincipalModel.USER, + permBits, + roleId: role!._id, + grantedBy, + ...(tenantId && { tenantId }), + }); +} + +function updateRole(accessRoleId = AccessRoleIds.AGENT_EDITOR) { + return service.bulkUpdateResourcePermissions({ + resourceType: ResourceType.AGENT, + resourceId, + updatedPrincipals: [{ type: PrincipalType.USER, id: userId.toString(), accessRoleId }], + grantedBy, + }); +} + +describe('role-only ACL writes', () => { + test.each([ + [AccessRoleIds.AGENT_VIEWER, RoleBits.VIEWER], + [AccessRoleIds.AGENT_EDITOR, RoleBits.EDITOR], + [AccessRoleIds.AGENT_OWNER, RoleBits.OWNER], + ])('replaces role bits for every valid mask with %s', async (roleId, roleBits) => { + const masks = permissionBitSupersets(0); + for (const mask of masks) { + await seed(mask); + } + const before = await entries.find(filter).sort({ _id: 1 }).lean(); + const result = await updateRole(roleId); + const after = await entries.find(filter).sort({ _id: 1 }).lean(); + expect(after).toHaveLength(before.length); + expect(after.map((entry) => entry.permBits)).toEqual( + before.map((entry) => (entry.permBits & ~RoleBits.OWNER) | roleBits), + ); + const role = await methods.findRoleByIdentifier(roleId); + expect(after.every((entry) => entry.roleId?.toString() === role!._id.toString())).toBe(true); + expect(result.insightsChanges).toEqual([]); + expect(result.errors).toEqual([]); + }); + + test.each([ + { type: PrincipalType.USER, id: userId.toString(), model: PrincipalModel.USER }, + { type: PrincipalType.GROUP, id: new Types.ObjectId().toString(), model: PrincipalModel.GROUP }, + { type: PrincipalType.ROLE, id: 'USER', model: PrincipalModel.ROLE }, + ])( + 'initializes and updates a $type principal without granting Insights', + async ({ type, id, model }) => { + for (const accessRoleId of [AccessRoleIds.AGENT_OWNER, AccessRoleIds.AGENT_VIEWER]) { + const result = await service.bulkUpdateResourcePermissions({ + resourceType: ResourceType.AGENT, + resourceId, + updatedPrincipals: [{ type, id, accessRoleId }], + grantedBy, + }); + expect(result.errors).toEqual([]); + expect(result.insightsChanges).toEqual([]); + } + const after = await entries.find({ resourceId, principalType: type }).lean(); + expect(after).toHaveLength(1); + expect(after[0].principalId?.toString()).toBe(id); + expect(after[0].principalModel).toBe(model); + expect(after[0].permBits).toBe(RoleBits.VIEWER); + }, + ); + + /** + * Intercept the write where it reaches the driver, not at a package method: + * the guarded write is issued from inside data-schemas, so a spy on the + * methods object silently observes nothing and every race test would pass + * while injecting nothing at all. + */ + function raceBeforeWrite(between: () => Promise, { once = true } = {}) { + const real = entries.bulkWrite.bind(entries) as (...args: never[]) => Promise; + const impl = (async (...args: never[]) => { + await between(); + return real(...args); + }) as unknown as typeof entries.bulkWrite; + const spy = jest.spyOn(entries, 'bulkWrite'); + return once ? spy.mockImplementationOnce(impl) : spy.mockImplementation(impl); + } + + test('emits one guarded $set per entry and an identity-only insert', async () => { + const spy = jest.spyOn(entries, 'bulkWrite'); + await updateRole(); + const insertOps = spy.mock.calls[spy.mock.calls.length - 1][0]; + await updateRole(); + const guardedOps = spy.mock.calls[spy.mock.calls.length - 1][0]; + + expect(await entries.countDocuments(filter)).toBe(1); + expect((await entries.findOne(filter))!.permBits).toBe(RoleBits.EDITOR); + + /** No entry yet: identity only, so a vanished grant is never inherited. */ + expect(insertOps).toHaveLength(1); + expect(insertOps[0]).toMatchObject({ + updateOne: { + filter, + update: { $setOnInsert: { permBits: RoleBits.EDITOR } }, + upsert: true, + }, + }); + + /** Entry exists: compare-and-set on the observed value, metadata included. */ + expect(guardedOps).toHaveLength(1); + const op = guardedOps[0] as { + updateOne: { + filter: Record; + update: { $set?: Record }; + upsert?: boolean; + }; + }; + expect(op.updateOne.filter).toHaveProperty('_id'); + expect(op.updateOne.filter.permBits).toBe(RoleBits.EDITOR); + expect(op.updateOne.upsert).not.toBe(true); + expect(op.updateOne.update).not.toHaveProperty('$bit'); + expect(Array.isArray(op.updateOne.update)).toBe(false); + expect(op.updateOne.update.$set).toEqual( + expect.objectContaining({ permBits: RoleBits.EDITOR, roleId: expect.anything() }), + ); + }); + + test.each([ + { label: 'grant', initial: 1, concurrent: 17, expected: 19 }, + { label: 'revocation', initial: 17, concurrent: 1, expected: 3 }, + ])( + 'preserves a concurrent Insights $label after the snapshot', + async ({ initial, concurrent, expected }) => { + await seed(initial); + raceBeforeWrite(async () => { + if (concurrent === null) await entries.deleteMany(filter); + else await entries.updateMany(filter, { $set: { permBits: concurrent } }); + }); + const result = await updateRole(); + expect((await entries.findOne(filter))!.permBits).toBe(expected); + expect(result.insightsChanges).toEqual([]); + expect(result.errors).toEqual([]); + }, + ); + + test('never filters a permission write on an enumerated mask list', async () => { + await seed(PermissionBits.VIEW | PermissionBits.VIEW_INSIGHTS); + const spy = jest.spyOn(entries, 'bulkWrite'); + await updateRole(); + /** + * The guard is the observed value, never a list of legal masks, so this + * write needs no bit-operator or mask-enumeration support from the engine. + */ + const payload = JSON.stringify(spy.mock.calls.map(([ops]) => ops)); + expect(payload).not.toContain('$in'); + expect(payload).not.toContain('$bit'); + expect((await entries.findOne(filter))!.permBits).toBe(19); + }); + + test('applies the role to an entry created after the read, without duplicating it', async () => { + raceBeforeWrite(() => seed(PermissionBits.VIEW | PermissionBits.VIEW_INSIGHTS)); + await updateRole(); + /** + * The identity upsert matched the entry that appeared instead of inserting, + * so the role had not been applied; that is a miss, and the retry applies it + * while keeping the Insights grant the other writer just made. + */ + expect(await entries.countDocuments(filter)).toBe(1); + expect((await entries.findOne(filter))!.permBits).toBe(19); + }); + + test('keeps explicit Insights upsert indices correct after expanded role-only writes', async () => { + await seed(17); + raceBeforeWrite(() => entries.deleteMany(filter)); + const result = await service.bulkUpdateResourcePermissions({ + resourceType: ResourceType.AGENT, + resourceId, + updatedPrincipals: [ + { + type: PrincipalType.USER, + id: new Types.ObjectId().toString(), + accessRoleId: AccessRoleIds.AGENT_EDITOR, + }, + { + type: PrincipalType.USER, + id: userId.toString(), + accessRoleId: AccessRoleIds.AGENT_EDITOR, + viewInsights: false, + }, + ], + grantedBy, + }); + expect(result.insightsChanges).toEqual([ + expect.objectContaining({ action: 'removed', previousEntry: null }), + ]); + await service.restoreInsightsPermissionChanges({ + resourceType: ResourceType.AGENT, + resourceId, + changes: result.insightsChanges, + }); + expect(await entries.findOne(filter)).toBeNull(); + expect(await entries.countDocuments({ resourceId })).toBe(1); + }); + + test('initializes missing legacy permission fields without inheriting Insights', async () => { + await seed(1); + await entries.collection.updateMany(filter, { $unset: { permBits: '' } }); + await updateRole(); + const after = await entries.find(filter).lean(); + expect(after).toHaveLength(1); + expect(after[0].permBits).toBe(RoleBits.EDITOR); + }); + + test('applies the role even when a preserved bit changes mid-write (#16170 review)', async () => { + await seed(RoleBits.OWNER | PermissionBits.VIEW_INSIGHTS); + raceBeforeWrite(async () => { + /** An admin revokes Insights between our read and our write. */ + await entries.updateMany(filter, { $set: { permBits: RoleBits.OWNER } }); + }); + const result = await service.bulkUpdateResourcePermissions({ + resourceType: ResourceType.AGENT, + resourceId, + updatedPrincipals: [ + { + type: PrincipalType.USER, + id: userId.toString(), + accessRoleId: AccessRoleIds.AGENT_VIEWER, + }, + ], + grantedBy, + }); + expect(result.errors).toEqual([]); + const entry = await entries.findOne(filter).lean(); + /** The downgrade lands: no retained owner bits, no resurrected Insights. */ + expect(entry!.permBits).toBe(RoleBits.VIEWER); + const role = await methods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER); + expect(entry!.roleId?.toString()).toBe(role!._id.toString()); + }); + + test('propagates failure before any write and permits an explicit retry', async () => { + await seed(RoleBits.OWNER); + await seed(RoleBits.OWNER | PermissionBits.VIEW_INSIGHTS); + jest + .spyOn(entries, 'bulkWrite') + .mockImplementationOnce((() => + Promise.reject( + new Error('injected write failure'), + )) as unknown as typeof entries.bulkWrite); + await expect(updateRole()).rejects.toThrow('injected write failure'); + const after = await entries.find(filter).lean(); + /** The whole batch failed, so both rows keep their pre-write state. */ + expect(after.map((entry) => entry.permBits).sort((a, b) => a - b)).toEqual([15, 31]); + await updateRole(); + const retried = await entries.find(filter).lean(); + expect(retried.map((entry) => entry.permBits).sort((a, b) => a - b)).toEqual([3, 19]); + }); + + test('scopes every partition and the identity upsert to the active tenant', async () => { + await seed(17, 'tenant-a'); + await seed(1, 'tenant-b'); + const result = await tenantStorage.run({ tenantId: 'tenant-a' }, async () => { + await methods.seedDefaultRoles(); + return updateRole(); + }); + expect(result.errors).toEqual([]); + const after = await entries.find(filter).sort({ tenantId: 1 }).lean(); + expect(after.map((entry) => [entry.tenantId, entry.permBits])).toEqual([ + ['tenant-a', 19], + ['tenant-b', 1], + ]); + }); + + test('does not resurrect an ACL deleted between read and write', async () => { + await seed(17); + const spy = raceBeforeWrite(() => entries.deleteMany(filter)); + await expect(updateRole()).rejects.toThrow('ACL deleted'); + expect(spy).toHaveBeenCalledTimes(1); + expect(await entries.countDocuments(filter)).toBe(0); + }); + + test('does not replay completed entries when a later duplicate conflicts', async () => { + await seed(1); + await seed(17); + const before = await entries.find(filter).sort({ _id: 1 }).lean(); + const real = entries.bulkWrite.bind(entries); + let calls = 0; + const intercept = (async (...args: Parameters) => { + calls++; + if (calls === 2) { + // Revoke the already-completed row, and make only the second CAS miss. + await entries.deleteOne({ _id: before[0]._id }); + await entries.updateOne({ _id: before[1]._id }, { $set: { permBits: 1 } }); + } + return real(...args); + }) as unknown as typeof entries.bulkWrite; + const spy = jest.spyOn(entries, 'bulkWrite').mockImplementation(intercept); + const result = await updateRole(); + expect(result.errors).toEqual([]); + expect(spy).toHaveBeenCalledTimes(3); + expect(await entries.findById(before[0]._id)).toBeNull(); + expect((await entries.findById(before[1]._id))!.permBits).toBe(3); + expect(await entries.countDocuments(filter)).toBe(1); + }); + + test('reports completed entries after a real partial failure without replaying them', async () => { + await seed(15); + await seed(31); + const real = entries.bulkWrite.bind(entries); + let calls = 0; + const intercept = (async (...args: Parameters) => { + if (++calls === 2) throw new Error('injected second-write failure'); + return real(...args); + }) as unknown as typeof entries.bulkWrite; + jest.spyOn(entries, 'bulkWrite').mockImplementation(intercept); + await expect(updateRole()).rejects.toMatchObject({ completedEntries: 1 }); + expect(calls).toBe(2); + const after = await entries.find(filter).sort({ _id: 1 }).lean(); + expect(after.map((entry) => entry.permBits)).toEqual([3, 31]); + const storedRole = await mongoose.models.AccessRole.findById(after[0].roleId).lean<{ + permBits: number; + }>(); + expect(storedRole!.permBits).toBe(3); + }); + + test('reads from primary even with a nontransactional secondaryPreferred session', async () => { + await seed(17); + const session = await mongoose.startSession(); + const find = jest.spyOn(entries, 'find'); + try { + await methods.replaceRoleBits( + [ + { + filter, + insert: {}, + roleBits: 3, + metadata: {}, + }, + ], + { session }, + ); + expect(session.inTransaction()).toBe(false); + expect(find.mock.results.length).toBeGreaterThan(0); + for (const result of find.mock.results) { + expect(result.value.getOptions().readPreference.mode).toBe('primary'); + } + } finally { + await session.endSession(); + } + }); + + test.each([NaN, Infinity, -1, 0, 1.5, 101])( + 'rejects invalid retry budget %s before writing', + async (maxAttempts) => { + const spy = jest.spyOn(entries, 'bulkWrite'); + await expect(methods.replaceRoleBits([], { maxAttempts })).rejects.toThrow(); + expect(spy).not.toHaveBeenCalled(); + }, + ); + + test('honors a supplied one-attempt budget', async () => { + await seed(17); + const spy = raceBeforeWrite(() => entries.updateMany(filter, { $set: { permBits: 1 } })); + await expect( + methods.replaceRoleBits([{ filter, insert: {}, roleBits: 3, metadata: {} }], { + maxAttempts: 1, + }), + ).rejects.toThrow('after 1 attempts'); + expect(spy).toHaveBeenCalledTimes(1); + expect((await entries.findOne(filter))!.permBits).toBe(1); + }); + + test.each([null, -1, 1.5, 4294967297])( + 'rejects malformed stored permissions %s without truncating', + async (permBits) => { + await seed(1); + await entries.collection.updateMany(filter, { $set: { permBits } }); + const spy = jest.spyOn(entries, 'bulkWrite'); + await expect(updateRole()).rejects.toThrow('Invalid permBits'); + expect(spy).not.toHaveBeenCalled(); + expect((await entries.collection.findOne(filter))!.permBits).toBe(permBits); + }, + ); + + test('serializes two overlapping add/remove requests without accumulating both grants', async () => { + await seed(1); + const spy = raceBeforeWrite(async () => { + const b = await methods.modifyPermissionBits( + PrincipalType.USER, + userId, + ResourceType.AGENT, + resourceId, + 4, + 2, + ); + expect(b!.permBits).toBe(5); + }); + const a = await methods.modifyPermissionBits( + PrincipalType.USER, + userId, + ResourceType.AGENT, + resourceId, + 2, + 4, + ); + expect(spy).toHaveBeenCalledTimes(3); + expect(a!.permBits).toBe(3); + expect((await entries.findOne(filter))!.permBits).toBe(3); + }); + + test('removal wins when add/remove masks overlap', async () => { + await seed(17); + const result = await methods.modifyPermissionBits( + PrincipalType.USER, + userId, + ResourceType.AGENT, + resourceId, + 3, + 2, + ); + expect(result!.permBits).toBe(17); + }); + + test('keeps all guarded writes inside a caller-owned transaction', async () => { + const replica = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + const connection = new mongoose.Mongoose(); + try { + await connection.connect(replica.getUri()); + createModels(connection); + const transactionMethods = createMethods(connection); + await transactionMethods.seedDefaultRoles(); + const transactionService = new AccessControlService(connection, transactionMethods); + const acl = connection.models.AclEntry as Model; + const role = await transactionMethods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER); + await acl.create({ + ...filter, + principalModel: PrincipalModel.USER, + permBits: 17, + roleId: role!._id, + grantedBy, + }); + /** + * `createModels` schedules autoIndex builds and the insert above creates + * the collection; both are catalog changes. A transaction that starts + * while one is still in flight aborts with a transient "Unable to write + * to collection ... due to catalog changes" error, so let the models this + * transaction touches finish building before opening the session. + */ + await Promise.all([acl.init(), connection.models.AccessRole.init()]); + const session = await connection.startSession(); + try { + session.startTransaction(); + const result = await transactionService.bulkUpdateResourcePermissions({ + resourceType: ResourceType.AGENT, + resourceId, + updatedPrincipals: [ + { + type: PrincipalType.USER, + id: userId.toString(), + accessRoleId: AccessRoleIds.AGENT_EDITOR, + }, + ], + grantedBy, + session, + }); + expect(result.errors).toEqual([]); + expect((await acl.findOne(filter).session(session))!.permBits).toBe(19); + await session.abortTransaction(); + expect((await acl.findOne(filter))!.permBits).toBe(17); + } finally { + await session.endSession(); + } + } finally { + await connection.disconnect(); + await replica.stop(); + } + }, 30000); + + test('keeps role bits and metadata consistent under two concurrent role updates', async () => { + await seed(RoleBits.OWNER | PermissionBits.VIEW_INSIGHTS); + const viewer = await methods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER); + raceBeforeWrite(() => + /** A competing Viewer request lands atomically between our read and write. */ + entries.updateMany(filter, { + $set: { + permBits: RoleBits.VIEWER | PermissionBits.VIEW_INSIGHTS, + roleId: viewer!._id, + }, + }), + ); + + const result = await updateRole(AccessRoleIds.AGENT_EDITOR); + expect(result.errors).toEqual([]); + + const entry = (await entries.findOne(filter).lean())!; + /** + * One logical winner: the stored role bits and the stored role reference + * cannot disagree. Resolved through the reference rather than compared to a + * freshly looked-up role, so repeated seeding cannot mask a divergence. + */ + expect(entry.permBits & RoleBits.OWNER).toBe(RoleBits.EDITOR); + const storedRole = await mongoose.models.AccessRole.findById(entry.roleId).lean<{ + accessRoleId: string; + permBits: number; + }>(); + expect(storedRole?.accessRoleId).toBe(AccessRoleIds.AGENT_EDITOR); + expect(entry.permBits & RoleBits.OWNER).toBe(storedRole!.permBits & RoleBits.OWNER); + expect(entry.permBits & PermissionBits.VIEW_INSIGHTS).toBe(PermissionBits.VIEW_INSIGHTS); + }); + + test('reports failure, not success, when the guard never holds', async () => { + await seed(RoleBits.OWNER | PermissionBits.VIEW_INSIGHTS); + const viewer = await methods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER); + let round = 0; + raceBeforeWrite( + () => + /** A different value every attempt, so no guard can ever match. */ + entries.updateMany(filter, { + $set: { + permBits: PermissionBits.VIEW_INSIGHTS | (round++ % 2 === 0 ? 1 : 2), + roleId: viewer!._id, + }, + }), + { once: false }, + ); + + await expect(updateRole()).rejects.toThrow(/permBits|attempt/i); + const entry = (await entries.findOne(filter).lean())!; + /** Failed loudly and left the competing writer's state intact. */ + expect(entry.roleId?.toString()).toBe(viewer!._id.toString()); + expect([17, 18]).toContain(entry.permBits); + }); + + test('preserves stored bits outside the known permission enum', async () => { + const role = await methods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER); + /** Written past the schema validator, as a legacy or external writer could. */ + await entries.collection.insertOne({ + ...filter, + principalModel: PrincipalModel.USER, + permBits: PermissionBits.VIEW | 64, + roleId: role!._id, + grantedBy, + grantedAt: new Date(), + }); + + await updateRole(); + + const entry = (await entries.findOne(filter).lean())!; + expect(entry.permBits & RoleBits.OWNER).toBe(RoleBits.EDITOR); + expect(entry.permBits & 64).toBe(64); + }); +}); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 15f78c3de3..969b330513 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -2905,8 +2905,12 @@ export const openIdDiscoverySchema = z.object({ export type TOpenIdDiscoveryConfig = z.infer; +/** Maximum CAS attempts per ACL document, including the initial attempt. */ +export const permissionWriteAttemptsSchema = z.number().int().min(1).max(100).default(3); + export const configSchema = z.object({ version: z.string(), + permissions: z.object({ maxWriteAttempts: permissionWriteAttemptsSchema }).optional(), cache: z.boolean().default(true), ocr: ocrSchema.optional(), webSearch: webSearchSchema.optional(), diff --git a/packages/data-provider/src/permissionWriteAttempts.spec.ts b/packages/data-provider/src/permissionWriteAttempts.spec.ts new file mode 100644 index 0000000000..1458ec85b6 --- /dev/null +++ b/packages/data-provider/src/permissionWriteAttempts.spec.ts @@ -0,0 +1,16 @@ +import { configSchema, permissionWriteAttemptsSchema } from './config'; + +describe('permission write retry budget', () => { + test('defaults once and retains an operator-supplied budget in librechat.yaml', () => { + expect(permissionWriteAttemptsSchema.parse(undefined)).toBe(3); + expect(configSchema.parse({ version: '1.3.1', permissions: {} }).permissions).toEqual({ + maxWriteAttempts: 3, + }); + expect( + configSchema.parse({ version: '1.3.1', permissions: { maxWriteAttempts: 7 } }).permissions, + ).toEqual({ maxWriteAttempts: 7 }); + }); + test.each([NaN, Infinity, -1, 0, 1.5, 101])('rejects %s', (value) => { + expect(permissionWriteAttemptsSchema.safeParse(value).success).toBe(false); + }); +}); diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 4a2a7f5a23..a642ab5eaf 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -18,6 +18,7 @@ export { defaultRate, createTxMethods, permissionBitSupersets, + PERM_BITS_WRITE_ATTEMPTS, partitionIssues, validateSkillName, validateSkillBody, diff --git a/packages/data-schemas/src/methods/aclEntry.ts b/packages/data-schemas/src/methods/aclEntry.ts index d80f0ae370..f36d670079 100644 --- a/packages/data-schemas/src/methods/aclEntry.ts +++ b/packages/data-schemas/src/methods/aclEntry.ts @@ -1,5 +1,10 @@ import { Types } from 'mongoose'; -import { PrincipalType, PrincipalModel, PermissionBits } from 'librechat-data-provider'; +import { + PrincipalType, + PrincipalModel, + PermissionBits, + permissionWriteAttemptsSchema, +} from 'librechat-data-provider'; import type { AnyBulkWriteOperation, ClientSession, @@ -10,6 +15,7 @@ import type { import type { IAclEntry } from '~/types'; import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite'; import { MAX_PERM_BITS } from '~/common/permissions'; +import { RoleBits } from '~/common/enum'; /** * Empty frozen array shared by every rejection path. Returning a single @@ -76,6 +82,25 @@ export function permissionBitSupersets(requiredBits: number): readonly number[] return frozen; } +/** + * Attempts for a guarded permission write before it reports failure. Callers + * may override per call; a conflict is only ever resolved by retrying, never by + * writing over a value the caller did not observe. + */ +export const PERM_BITS_WRITE_ATTEMPTS: number = permissionWriteAttemptsSchema.parse(undefined); + +/** One principal's role replacement, as requested by the caller. */ +export type RoleBitsWrite = { + /** Identity of the principal on the resource; never filters on permissions. */ + filter: Record; + /** Identity fields used only when the entry has to be created. */ + insert: Record; + /** The requested role's bits; non-role bits are masked off. */ + roleBits: number; + /** Role metadata that must land in the SAME write as the role bits. */ + metadata: Record; +}; + export function createAclEntryMethods(mongoose: typeof import('mongoose')): { findEntriesByPrincipal: ( principalType: string, @@ -134,6 +159,7 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): { addBits?: number | null, removeBits?: number | null, session?: ClientSession, + maxAttempts?: number, ) => Promise; findAccessibleResources: ( principalsList: Array<{ principalType: string; principalId?: string | Types.ObjectId }>, @@ -146,6 +172,10 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): { filter: Record, options?: { session?: ClientSession }, ) => Promise; + replaceRoleBits: ( + writes: RoleBitsWrite[], + options?: { session?: ClientSession; maxAttempts?: number }, + ) => Promise<{ upsertedWrites: Set }>; bulkWriteAclEntries: ( ops: AnyBulkWriteOperation[], options?: { session?: ClientSession }, @@ -453,6 +483,7 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): { addBits?: number | null, removeBits?: number | null, session?: ClientSession, + maxAttempts?: number, ): Promise { const AclEntry = mongoose.models.AclEntry as Model; const query: Record = { @@ -468,26 +499,20 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): { : principalId; } - const update: Record = {}; - - if (addBits) { - update.$bit = { permBits: { or: addBits } }; - } - - if (removeBits) { - if (!update.$bit) { - update.$bit = {}; - } - const bitUpdate = update.$bit as Record; - bitUpdate.permBits = { ...(bitUpdate.permBits as Record), and: ~removeBits }; - } - - const options = { - new: true, - ...(session ? { session } : {}), - }; - - return await AclEntry.findOneAndUpdate(query, update, options); + const addMask = permissionMask(addBits ?? 0); + const removeMask = permissionMask(removeBits ?? 0); + const attempts = permissionWriteAttemptsSchema.parse(maxAttempts); + const [entry] = await readPermissionEntries(query, session); + if (entry == null) return null; + if (addMask === 0 && removeMask === 0) return AclEntry.hydrate(entry); + return mutatePermissionEntry( + entry, + query, + (bits) => (bits | addMask) & ~removeMask, + {}, + attempts, + session, + ); } /** @@ -543,6 +568,135 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): { return AclEntry.deleteMany(filter, options || {}); } + /** Reject values JavaScript bit operations would truncate. Missing legacy fields mean zero. */ + function permissionMask(value: unknown): number { + if (value === undefined) return 0; + if (typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 0x7fffffff) { + throw new Error('Invalid permBits: expected a nonnegative 31-bit integer'); + } + return value; + } + + /** Pin nontransactional reads to the write authority, including explicit sessions. */ + async function readPermissionEntries(filter: Record, session?: ClientSession) { + const AclEntry = mongoose.models.AclEntry as Model; + const query = AclEntry.find(filter, null, session ? { session } : {}).sort({ _id: 1 }); + if (!session?.inTransaction()) query.read('primary'); + return query.lean(); + } + + /** + * The single permission mutation primitive. One CAS changes bits and metadata + * together. Retry only THIS document, never documents that already completed. + * Deletion wins over a pending mutation: never retarget it to a replacement ACL. + * Transaction conflicts and ambiguous write failures propagate to the caller; + * only a definite zero-match result is retried here. + */ + async function mutatePermissionEntry( + initial: Awaited>[number], + identity: Record, + transform: (bits: number) => number, + metadata: Record, + maxAttempts: number, + session?: ClientSession, + ): Promise { + const AclEntry = mongoose.models.AclEntry as Model; + let current: typeof initial | undefined = initial; + for (let attempt = 0; attempt < maxAttempts; attempt++) { + if (current == null) return null; + const observed = current.permBits; + const permBits = permissionMask(transform(permissionMask(observed))); + const guard: Record = { ...identity, _id: current._id }; + // Bind the guard to the snapshot's role/audit state as well as its bits. + for (const key of ['permBits', 'roleId', 'grantedBy', 'grantedAt']) { + const value = (current as unknown as Record)[key]; + guard[key] = value === undefined ? { $exists: false } : value; + } + const result = await bulkWriteAclEntries( + [{ updateOne: { filter: guard, update: { $set: { ...metadata, permBits } } } }], + session ? { session } : {}, + ); + if (result.matchedCount === 1) { + return AclEntry.hydrate({ ...current, ...metadata, permBits }); + } + [current] = await readPermissionEntries({ ...identity, _id: initial._id }, session); + } + if (current == null) return null; + throw new Error( + `Permission write conflict: permBits guard did not hold after ${maxAttempts} attempts`, + ); + } + + /** + * Process each observed ACL once, including duplicates. Completion is tracked + * per document; a later failure may leave earlier writes committed on standalone + * databases and reports that fact without replaying them. This is not a batch transaction. + */ + async function replaceRoleBits( + writes: RoleBitsWrite[], + options?: { session?: ClientSession; maxAttempts?: number }, + ): Promise<{ upsertedWrites: Set }> { + const maxAttempts = permissionWriteAttemptsSchema.parse(options?.maxAttempts); + const session = options?.session; + const upsertedWrites = new Set(); + let completedEntries = 0; + try { + for (const [index, write] of writes.entries()) { + const roleOnly = permissionMask(write.roleBits) & RoleBits.OWNER; + let existing = await readPermissionEntries(write.filter, session); + if (existing.length === 0) { + const result = await bulkWriteAclEntries( + [ + { + updateOne: { + filter: write.filter, + update: { + $setOnInsert: { ...write.insert, ...write.metadata, permBits: roleOnly }, + }, + upsert: true, + }, + }, + ], + session ? { session } : {}, + ); + if (result.upsertedCount === 1) { + upsertedWrites.add(index); + completedEntries++; + continue; + } + // Another writer created this identity first. Read and mutate that row, + // rather than claiming that a no-op identity upsert applied our role. + existing = await readPermissionEntries(write.filter, session); + if (existing.length === 0) throw new Error('ACL disappeared during role insertion'); + } + for (const entry of existing) { + const updated = await mutatePermissionEntry( + entry, + write.filter, + (bits) => (bits & ~RoleBits.OWNER) | roleOnly, + write.metadata, + maxAttempts, + session, + ); + if (updated == null) + throw new Error('ACL deleted during role update; permission was not recreated'); + completedEntries++; + } + } + return { upsertedWrites }; + } catch (error) { + // Preserve MongoDB error labels so the owner can retry/abort its transaction. + if (error instanceof Error) { + Object.assign(error, { + completedEntries, + transactional: session?.inTransaction() === true, + }); + error.message += ` (${completedEntries} ACL writes completed before failure; transaction may roll back)`; + } + throw error; + } + } + /** * Performs a bulk write operation on ACL entries. * @param ops - Array of bulk write operations @@ -654,6 +808,7 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): { modifyPermissionBits, findAccessibleResources, deleteAclEntries, + replaceRoleBits, bulkWriteAclEntries, findPublicResourceIds, aggregateAclEntries, diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index 316bdd081d..bce00d7fa2 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -42,7 +42,12 @@ import { type UserGroupMethods, type UserGroupDeps, } from './userGroup'; -import { createAclEntryMethods, permissionBitSupersets, type AclEntryMethods } from './aclEntry'; +import { + createAclEntryMethods, + permissionBitSupersets, + PERM_BITS_WRITE_ATTEMPTS, + type AclEntryMethods, +} from './aclEntry'; import { createSystemGrantMethods, type SystemGrantMethods } from './systemGrant'; import { createAuditLogMethods, @@ -197,7 +202,7 @@ export { digestMCPAuthorityValue, }; export { tokenValues, cacheTokenValues, premiumTokenValues, defaultRate, createTxMethods }; -export { permissionBitSupersets }; +export { permissionBitSupersets, PERM_BITS_WRITE_ATTEMPTS }; export { CLIENT_MESSAGE_SELECT, SUBAGENT_TRANSCRIPT_SOURCE_BYTE_LIMIT }; export { partitionIssues,