fix: Guard transactional channel writes (#17770)

This commit is contained in:
Dustin Healy 2026-11-10 21:04:07 -08:00 • committed by GitHub
parent b83bcdc572
commit 288175e77a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 414 additions and 9 deletions

View file

@ -7,6 +7,7 @@ import type {
IAgentApiKey,
AgentApiKeyBinding,
} from '~/types';
import { startRequiredTransactionSession } from '~/utils/transactions';
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { prepareChannelOwnerAdmission } from './channelOwner';
import { createChannelIndexGate } from './channelIndexes';
@ -89,7 +90,7 @@ export function createAgentApiKeyMethods(mongoose: typeof import('mongoose')): {
userId: String(data.userId),
tenantId,
});
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
const created = await session.withTransaction(async () => {
if (!(await admit(session))) throw new Error('Bound-key owner unavailable');

View file

@ -0,0 +1,289 @@
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import { MongoMemoryServer, MongoMemoryReplSet } from 'mongodb-memory-server';
import { withChannelFeedbackAuthority } from './channelFeedbackScope';
import { TransactionUnavailableError } from '~/utils/transactions';
import { createChannelIdentityStorage } from './channelIdentity';
import { createAgentApiKeyMethods } from './agentApiKey';
import { createChannelRunStorage } from './channelRun';
import { createAgentChannelMethods } from './channel';
import { retireAgentChannel } from './channelScope';
import { createModels } from '~/models';
jest.mock('~/config/winston', () => ({
debug: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
}));
describe('transactional channel writes fail before private persistence', () => {
const standalone = new mongoose.Mongoose();
const ferret = process.env.FERRETDB_URI ? new mongoose.Mongoose() : undefined;
const engines: Array<readonly [string, typeof standalone]> = [
['standalone', standalone],
...(ferret ? [['FerretDB', ferret] as const] : []),
];
let mongo: MongoMemoryServer;
beforeAll(async () => {
mongo = await MongoMemoryServer.create();
await standalone.connect(mongo.getUri(), { autoIndex: false });
if (ferret)
await ferret.connect(process.env.FERRETDB_URI!, {
dbName: `guards_${randomUUID().replace(/-/g, '')}`,
autoIndex: false,
});
for (const [, db] of engines) createModels(db);
}, 60000);
afterAll(async () => {
await Promise.all(engines.map(([, db]) => db.disconnect()));
await mongo?.stop();
});
describe.each(engines)('%s', (_name, db) => {
it('rejects channel credentials, identity, run, bound-key and feedback writes before payload insertion', async () => {
const userId = new db.Types.ObjectId();
const channelId = randomUUID();
const channels = createAgentChannelMethods(db);
await expect(
channels.createAgentChannel({
channelId,
agentId: 'agent',
createdBy: String(userId),
type: 'slack',
status: 'active',
credentialMode: 'shared',
revision: 'revision',
encryptedBotToken: 'test-ciphertext',
encryptedSigningSecret: 'test-ciphertext',
slack: { teamId: 'team', appId: 'app', botUserId: 'bot' },
}),
).rejects.toBeInstanceOf(TransactionUnavailableError);
await expect(
createChannelIdentityStorage(db).saveChannelIdentity(
{
tenantId: '_',
linkId: 'link',
revision: 'revision',
state: 'pending',
ticketHash: 'test-ticket',
ticketExpiresAtMs: 1000,
expiresAtMs: 2000,
identity: {
tenantId: '_',
channelId,
channelRevision: 'revision',
teamId: 'team',
externalUserId: 'actor',
agentId: 'agent',
},
},
null,
),
).rejects.toBeInstanceOf(TransactionUnavailableError);
await expect(
createChannelRunStorage(db).admit(
{
tenantId: '_',
channelId,
agentId: 'agent',
principal: { userId: String(userId), channelRevision: 'revision' },
admission: {
eventId: 'event',
externalChannelId: 'channel',
externalThreadKey: 'thread',
externalMessageKey: 'message',
actor: { teamId: 'team', appId: 'app', externalUserId: 'actor' },
input: 'test private prompt',
},
},
Date.now(),
),
).rejects.toBeInstanceOf(TransactionUnavailableError);
await expect(
createAgentApiKeyMethods(db).createAgentApiKey({
userId,
name: 'Bound test key',
binding: { kind: 'channel', id: channelId },
}),
).rejects.toBeInstanceOf(TransactionUnavailableError);
const effect = jest.fn();
await expect(
withChannelFeedbackAuthority(
db,
{
tenantId: '_',
channelId,
channelRevision: 'revision',
teamId: 'team',
agentId: 'agent',
runId: 'run',
responseId: 'resp_test',
messageId: 'resp_test',
conversationId: randomUUID(),
principalId: String(userId),
externalMessageKey: 'message',
traceId: 'trace',
centralTraceExportEnabled: false,
},
effect,
),
).rejects.toBeInstanceOf(TransactionUnavailableError);
expect(effect).not.toHaveBeenCalled();
for (const model of [
'AgentChannel',
'ChannelIdentity',
'ChannelRun',
'ChannelLane',
'AgentApiKey',
'ChannelReply',
'ChannelFeedback',
'ChannelFeedbackEvent',
]) {
expect(await db.model(model).countDocuments()).toBe(0);
}
});
it('keeps ordinary personal API keys and missing-channel cleanup available', async () => {
const userId = new db.Types.ObjectId();
const keys = createAgentApiKeyMethods(db);
const created = await keys.createAgentApiKey({ userId, name: 'Personal test key' });
const validated = await keys.validateAgentApiKey(created.key);
expect(String(validated?.userId)).toBe(String(userId));
await db.model('AgentApiKey').deleteOne({ _id: created.id });
expect(
await retireAgentChannel(db, {
channelId: randomUUID(),
createdBy: String(userId),
}),
).toBeNull();
});
});
});
describe('channel transactions with a secondaryPreferred client default', () => {
const db = new mongoose.Mongoose();
let mongo: MongoMemoryReplSet;
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } });
await db.connect(mongo.getUri(), { readPreference: 'secondaryPreferred', autoIndex: false });
createModels(db);
}, 60000);
afterAll(async () => {
await db.disconnect();
await mongo?.stop();
});
it('commits channel, link, run, credential, feedback and retirement operations', async () => {
const userId = new db.Types.ObjectId();
const channelId = randomUUID();
const channels = createAgentChannelMethods(db);
await channels.createAgentChannel({
channelId,
agentId: 'agent',
createdBy: String(userId),
type: 'slack',
status: 'active',
credentialMode: 'shared',
revision: 'revision',
encryptedBotToken: 'test-ciphertext',
encryptedSigningSecret: 'test-ciphertext',
slack: { teamId: 'team', appId: 'app', botUserId: 'bot' },
});
expect(
await channels.updateAgentChannel(channelId, String(userId), { revision: 'revision' }),
).toMatchObject({ channelId, status: 'active' });
const identities = createChannelIdentityStorage(db);
expect(
await identities.saveChannelIdentity(
{
tenantId: '_',
linkId: 'link',
revision: 'revision',
state: 'pending',
ticketHash: 'test-ticket',
ticketExpiresAtMs: Date.now() + 60000,
expiresAtMs: Date.now() + 120000,
identity: {
tenantId: '_',
channelId,
channelRevision: 'revision',
teamId: 'team',
externalUserId: 'actor',
agentId: 'agent',
},
},
null,
),
).toBe(true);
await identities.revokeChannelIdentities('_', channelId, 'revoked');
expect(await identities.readChannelLink('_', 'link')).toMatchObject({ state: 'revoked' });
const runs = createChannelRunStorage(db);
const { run } = await runs.admit(
{
tenantId: '_',
channelId,
agentId: 'agent',
principal: { userId: String(userId), channelRevision: 'revision' },
admission: {
eventId: 'event',
externalChannelId: 'channel',
externalThreadKey: 'thread',
externalMessageKey: 'message',
actor: { teamId: 'team', appId: 'app', externalUserId: 'actor' },
input: 'test prompt',
},
},
Date.now(),
);
expect(await runs.claim('_', channelId, 'worker', Date.now(), 30000)).toMatchObject({
runId: run.runId,
leaseOwner: 'worker',
});
const keys = createAgentApiKeyMethods(db);
const key = await keys.createAgentApiKey({
userId,
name: 'Bound test key',
binding: { kind: 'channel', id: channelId },
});
expect(await keys.validateAgentApiKey(key.key)).toBeNull();
expect(await keys.validateBoundAgentApiKey(key.key)).toMatchObject({ userId: String(userId) });
const conversationId = randomUUID();
await db.models.Message.create({
user: String(userId),
conversationId,
messageId: 'response',
isCreatedByUser: false,
sender: 'Agent',
endpoint: 'agents',
model: 'agent',
text: 'answer',
});
const mapping = {
tenantId: '_',
channelId,
channelRevision: 'revision',
teamId: 'team',
agentId: 'agent',
runId: run.runId,
responseId: 'response',
messageId: 'response',
conversationId,
principalId: String(userId),
externalMessageKey: 'reply',
traceId: 'trace',
centralTraceExportEnabled: false,
};
expect(await withChannelFeedbackAuthority(db, mapping, async () => 'authorized')).toBe(
'authorized',
);
expect(
await retireAgentChannel(db, { channelId, createdBy: String(userId) }, true),
).toMatchObject({ status: 'deleting' });
const retired = await db.models.AgentChannel.findOne({ channelId })
.select('+encryptedBotToken +encryptedSigningSecret')
.lean();
expect(retired).not.toHaveProperty('encryptedBotToken');
expect(retired).not.toHaveProperty('encryptedSigningSecret');
});
});

View file

@ -4,6 +4,7 @@ import type { ClientSession } from 'mongoose';
import type { AgentChannelRecord } from '~/models/channel';
import { retireAgentChannel, initializeChannelScope, touchChannelScope } from './channelScope';
import { eraseChannelFeedbackForChannel } from './channelFeedbackErasure';
import { startRequiredTransactionSession } from '~/utils/transactions';
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { prepareChannelOwnerAdmission } from './channelOwner';
import { createChannelIndexGate } from './channelIndexes';
@ -72,7 +73,7 @@ export function createAgentChannelMethods(
write: (session: ClientSession) => Promise<T>,
): Promise<T> {
const admit = await prepareChannelOwnerAdmission(mongoose, { userId: createdBy, tenantId });
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
let result: T | undefined;
await session.withTransaction(async () => {

View file

@ -2,6 +2,7 @@ import type { ChannelReplyMapping } from 'librechat-data-provider';
import type { ClientSession } from 'mongoose';
import { initializeResponseScopes, responseScopeKeys } from './responseScope';
import { assertChannelTenant, touchChannelAuthority } from './channelScope';
import { startRequiredTransactionSession } from '~/utils/transactions';
import { SERVER_AUTHORED_MESSAGE_FILTER } from '~/utils/authored';
import { createResponseScopeModel } from '~/models/responseScope';
@ -131,7 +132,7 @@ export async function withChannelFeedbackAuthority<T>(
};
const keys = responseScopeKeys(owner, mapping.conversationId);
await initializeResponseScopes(mongoose, keys);
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
let value!: T;
await session.withTransaction(async () => {

View file

@ -7,6 +7,7 @@ import {
touchChannelAuthority,
lockChannelScope,
} from './channelScope';
import { startRequiredTransactionSession } from '~/utils/transactions';
import { prepareChannelOwnerAdmission } from './channelOwner';
import { createChannelIndexGate } from './channelIndexes';
export type { ChannelIdentityRecord } from '~/models/channelIdentity';
@ -119,7 +120,7 @@ export function createChannelIdentityStorage(
tenantId: record.tenantId === '_' ? undefined : record.tenantId,
})
: undefined;
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
let saved = false;
await session.withTransaction(async () => {
@ -172,7 +173,7 @@ export function createChannelIdentityStorage(
const record = await model().findOne(filter).select('identity.channelId').lean();
if (!record) return false;
await initializeChannelScope(mongoose, owner.tenantId, record.identity.channelId);
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
let matched = false;
await session.withTransaction(async () => {
@ -193,7 +194,7 @@ export function createChannelIdentityStorage(
async revokeChannelIdentities(tenantId, channelId, revision) {
assertChannelTenant(tenantId);
await initializeChannelScope(mongoose, tenantId, channelId);
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
await session.withTransaction(async () => {
await lockChannelScope(mongoose, tenantId, channelId, session);

View file

@ -30,6 +30,7 @@ import {
ChannelRunConflict,
} from './channelProtocol';
import { initializeChannelScope, touchChannelAuthority, touchChannelScope } from './channelScope';
import { startRequiredTransactionSession } from '~/utils/transactions';
import { prepareChannelOwnerAdmission } from './channelOwner';
import { createChannelIndexGate } from './channelIndexes';
import { responseOwnerFilter } from './responseScope';
@ -215,7 +216,7 @@ export function createChannelRunStorage(
const ensureIndexes = createChannelIndexGate(() => [runs(), lanes()]);
async function transaction<T>(work: (session: ClientSession) => Promise<T>): Promise<T> {
await ensureIndexes();
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
let result: T | undefined;
await session.withTransaction(async () => {

View file

@ -1,6 +1,7 @@
import { createHash } from 'node:crypto';
import type { ClientSession } from 'mongoose';
import type { AgentChannelRecord } from '~/models/channel';
import { startRequiredTransactionSession } from '~/utils/transactions';
import { getTenantId, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { createChannelIdentityModel } from '~/models/channelIdentity';
import { createChannelScopeModel } from '~/models/channelScope';
@ -181,8 +182,19 @@ export async function retireAgentChannel(
redactCredentials = false,
): Promise<AgentChannelRecord | null> {
const { tenantId, channelId, createdBy } = connection;
assertChannelTenant(tenantId);
if (
!(await createAgentChannelModel(mongoose)
.exists({
channelId,
createdBy,
tenantId: tenantId || { $in: [null, ''] },
})
.read('primary'))
)
return null;
await initializeChannelScope(mongoose, tenantId, channelId);
const session = await mongoose.startSession();
const session = await startRequiredTransactionSession(mongoose);
try {
const deleting = await session.withTransaction(async () => {
const record = await createAgentChannelModel(mongoose)

View file

@ -1,6 +1,11 @@
import mongoose from 'mongoose';
import { MongoMemoryServer, MongoMemoryReplSet } from 'mongodb-memory-server';
import { getTransactionSupport, supportsTransactions } from './transactions';
import {
getTransactionSupport,
supportsTransactions,
startRequiredTransactionSession,
TransactionUnavailableError,
} from './transactions';
jest.mock('~/config/winston', () => ({ debug: jest.fn() }));
@ -131,4 +136,64 @@ describe('transaction capability against real MongoDB', () => {
jest.spyOn(replica.connection.db!, 'createCollection').mockRejectedValueOnce(error);
await expect(supportsTransactions(replica)).rejects.toBe(error);
});
it('reuses a required-session verdict instead of probing every subsequent transition', async () => {
const db = new mongoose.Mongoose();
await db.connect(repl.getUri(), { readPreference: 'secondaryPreferred' });
const starts = jest.spyOn(db, 'startSession');
try {
const first = await startRequiredTransactionSession(db);
await first.endSession();
const second = await startRequiredTransactionSession(db);
await second.endSession();
expect(starts).toHaveBeenCalledTimes(3); // One probe and two actual owned sessions.
expect(starts).toHaveBeenLastCalledWith({
defaultTransactionOptions: { readPreference: 'primary' },
});
} finally {
await db.disconnect();
}
});
it('retries a required-session probe after an operational failure', async () => {
const db = new mongoose.Mongoose();
await db.connect(repl.getUri());
const failure = new db.mongo.MongoNetworkError('test probe unavailable');
const starts = jest.spyOn(db, 'startSession').mockRejectedValueOnce(failure);
try {
await expect(startRequiredTransactionSession(db)).rejects.toBe(failure);
const session = await startRequiredTransactionSession(db);
await session.endSession();
expect(starts).toHaveBeenCalledTimes(3);
} finally {
await db.disconnect();
}
});
it('rechecks a required-session verdict when the connection gets a different client', async () => {
const db = new mongoose.Mongoose();
try {
await db.connect(mongo.getUri());
await expect(startRequiredTransactionSession(db)).rejects.toBeInstanceOf(
TransactionUnavailableError,
);
await db.disconnect();
await db.connect(repl.getUri(), { readPreference: 'secondaryPreferred' });
const session = await startRequiredTransactionSession(db);
await session.endSession();
} finally {
await db.disconnect();
}
});
it('does not open feature sessions after confirmed lack of transaction support', async () => {
const starts = jest.spyOn(standalone, 'startSession');
await expect(startRequiredTransactionSession(standalone)).rejects.toBeInstanceOf(
TransactionUnavailableError,
);
await expect(startRequiredTransactionSession(standalone)).rejects.toBeInstanceOf(
TransactionUnavailableError,
);
expect(starts).toHaveBeenCalledTimes(1);
});
});

View file

@ -1,3 +1,4 @@
import type { ClientSession, Connection } from 'mongoose';
import logger from '~/config/winston';
export const CANCEL_RATE = 1.15;
@ -77,3 +78,36 @@ export const getTransactionSupport = async (
}
return probe;
};
export class TransactionUnavailableError extends Error {
readonly code = 'transactions_unavailable';
constructor() {
super('This operation requires transactional storage');
this.name = 'TransactionUnavailableError';
}
}
const requiredTransactionSupport = new WeakMap<
Connection,
{
client: ReturnType<Connection['getClient']>;
supported: boolean | null;
}
>();
/** Probe before private writes: some unsupported engines fail only at commit. */
export async function startRequiredTransactionSession(
mongoose: typeof import('mongoose'),
): Promise<ClientSession> {
const connection = mongoose.connection;
const client = connection.getClient();
let state = requiredTransactionSupport.get(connection);
if (!state || state.client !== client) {
state = { client, supported: null };
requiredTransactionSupport.set(connection, state);
}
// Assignment happens only on a confirmed verdict; operational failures remain retryable.
state.supported = await getTransactionSupport(mongoose, state.supported);
if (!state.supported) throw new TransactionUnavailableError();
return mongoose.startSession({ defaultTransactionOptions: { readPreference: 'primary' } });
}