🗄️ perf: Cut Serial and Redundant Database Reads (#16711)

* perf: Cut Serial and Redundant Database Reads

* perf: Pick the Earliest Balance per User in MongoDB
This commit is contained in:
Marco Beretta 2026-10-03 16:41:47 +02:00 • committed by GitHub
parent b24226672d
commit 6f4e36665e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 59 additions and 35 deletions

View file

@ -2129,8 +2129,10 @@ const revertAgentVersionHandler = async (req, res) => {
*/
const getAgentCategories = async (_req, res) => {
try {
const categories = await db.getCategoriesWithCounts();
const promotedCount = await db.countPromotedAgents();
const [categories, promotedCount] = await Promise.all([
db.getCategoriesWithCounts(),
db.countPromotedAgents(),
]);
const formattedCategories = categories.map((category) => ({
value: category.value,
label: category.label,

View file

@ -2090,6 +2090,7 @@ describe('Convos Routes', () => {
context: `POST /api/convos/archive ${mockConversationId}`,
preserveUpdatedAt: true,
noUpsert: true,
appendMessageIds: [],
},
);
});
@ -2123,6 +2124,7 @@ describe('Convos Routes', () => {
context: `POST /api/convos/archive ${mockConversationId}`,
preserveUpdatedAt: true,
noUpsert: true,
appendMessageIds: [],
},
);
});

View file

@ -661,6 +661,8 @@ router.post('/archive', validateConvoAccess, async (req, res) => {
preserveUpdatedAt: true,
/** Without timestamps, an upsert would insert a conversation that has none. */
noUpsert: true,
/** Metadata-only: skip rebuilding `messages` so a concurrent append is not erased. */
appendMessageIds: [],
},
);
@ -759,7 +761,7 @@ router.post('/update', validateConvoAccess, configMiddleware, async (req, res) =
interfaceConfig: req?.config?.interfaceConfig,
},
{ conversationId, title: sanitizedTitle },
{ context: `POST /api/convos/update ${conversationId}` },
{ context: `POST /api/convos/update ${conversationId}`, appendMessageIds: [] },
);
res.status(201).json(dbResponse);
} catch (error) {

View file

@ -15,9 +15,16 @@ const connect = require('./connect');
console.purple('Show the balance of all users');
console.purple('-----------------------------');
let users = await User.find({});
const users = await User.find({});
/** Earliest balance per user, matching the previous per-user `findOne().sort({ _id: 1 })` */
const balances = await Balance.aggregate([
{ $match: { user: { $in: users.map((user) => user._id) } } },
{ $sort: { _id: 1 } },
{ $group: { _id: '$user', tokenCredits: { $first: '$tokenCredits' } } },
]);
const balanceByUser = new Map(balances.map((balance) => [String(balance._id), balance]));
for (const user of users) {
let balance = await Balance.findOne({ user: user._id }).sort({ _id: 1 });
const balance = balanceByUser.get(String(user._id)) ?? null;
if (balance !== null) {
console.green(`User ${user.name} (${user.email}) has a balance of ${balance.tokenCredits}`);
} else {

View file

@ -15,19 +15,21 @@ const connect = require('./connect');
console.purple('Show the stats of all users');
console.purple('-----------------------------');
let users = await User.find({});
let userData = [];
for (const user of users) {
let conversationsCount = (await Conversation.countDocuments({ user: user._id })) ?? 0;
let messagesCount = (await Message.countDocuments({ user: user._id })) ?? 0;
userData.push({
User: user.name,
Email: user.email,
Conversations: conversationsCount,
Messages: messagesCount,
});
}
const countByUser = async (Model) => {
const rows = await Model.aggregate([{ $group: { _id: '$user', count: { $sum: 1 } } }]);
return new Map(rows.map(({ _id, count }) => [String(_id), count]));
};
const [users, conversationCounts, messageCounts] = await Promise.all([
User.find({}),
countByUser(Conversation),
countByUser(Message),
]);
const userData = users.map((user) => ({
User: user.name,
Email: user.email,
Conversations: conversationCounts.get(String(user._id)) ?? 0,
Messages: messageCounts.get(String(user._id)) ?? 0,
}));
userData.sort((a, b) => {
if (a.Conversations !== b.Conversations) {

View file

@ -79,7 +79,11 @@ export function createAgentApiKeyMethods(mongoose: typeof import('mongoose')): {
return null;
}
await AgentApiKey.updateOne({ _id: keyDoc._id }, { $set: { lastUsedAt: new Date() } });
AgentApiKey.updateOne({ _id: keyDoc._id }, { $set: { lastUsedAt: new Date() } }).catch(
(error: unknown) => {
logger.error('[validateAgentApiKey] Error updating lastUsedAt:', error);
},
);
return {
userId: keyDoc.userId,

View file

@ -44,13 +44,15 @@ export function createAgentCategoryMethods(mongoose: typeof import('mongoose')):
async function getCategoriesWithCounts(): Promise<(IAgentCategory & { agentCount: number })[]> {
const Agent = mongoose.models.Agent;
const categoryCounts = await Agent.aggregate([
{ $match: { category: { $exists: true, $ne: null } } },
{ $group: { _id: '$category', count: { $sum: 1 } } },
const [categoryCounts, categories] = await Promise.all([
Agent.aggregate([
{ $match: { category: { $exists: true, $ne: null } } },
{ $group: { _id: '$category', count: { $sum: 1 } } },
]),
getActiveCategories(),
]);
const countMap = new Map(categoryCounts.map((c) => [c._id, c.count]));
const categories = await getActiveCategories();
return categories.map((category) => ({
...category,

View file

@ -411,7 +411,7 @@ export function createConversationTagMethods(mongoose: typeof import('mongoose')
const ConversationTag = mongoose.models.ConversationTag as Model<IConversationTag>;
const Conversation = mongoose.models.Conversation;
const conversation = await Conversation.findOne({ user, conversationId }).lean();
const conversation = await Conversation.findOne({ user, conversationId }, 'tags').lean();
if (!conversation) {
throw new Error('Conversation not found');
}
@ -455,15 +455,13 @@ export function createConversationTagMethods(mongoose: typeof import('mongoose')
await tenantSafeBulkWrite(ConversationTag, bulkOps);
}
const updatedConversation = (
await Conversation.findOneAndUpdate(
{ user, conversationId },
{ $set: { tags: [...newTags] } },
{ new: true },
)
).toObject();
const updatedConversation = await Conversation.findOneAndUpdate(
{ user, conversationId },
{ $set: { tags: [...newTags] } },
{ new: true, projection: { tags: 1 } },
).lean<{ tags: string[] }>();
return updatedConversation.tags;
return (updatedConversation as { tags: string[] }).tags;
} catch (error) {
logger.error('[updateTagsForConversation] Error updating tags', error);
throw new Error('Error updating tags for conversation');

View file

@ -4856,10 +4856,15 @@ export function createAgentTriggerDeliveryMethods(
.lean<IAgentTriggerUserPurge[]>();
if (activity != null && markers.length > 0) activity.found = true;
let recovered = 0;
const users =
markers.length > 0
? await mongoose.models.User.find({ _id: { $in: markers.map((marker) => marker._id) } })
.select('agentTriggerDeletionStartedAt')
.lean<Array<{ _id: Types.ObjectId; agentTriggerDeletionStartedAt?: Date }>>()
: [];
const usersById = new Map(users.map((found) => [String(found._id), found]));
for (const marker of markers) {
const user = await mongoose.models.User.findById(marker._id)
.select('agentTriggerDeletionStartedAt')
.lean<{ agentTriggerDeletionStartedAt?: Date }>();
const user = usersById.get(String(marker._id)) ?? null;
if (user != null) {
if (user.agentTriggerDeletionStartedAt?.getTime() === marker.fenceStartedAt.getTime()) {
// Move a still-owned pre-commit marker behind this bounded scan so