From 6f4e36665e11015fd9b138fe46edc22d4aa64789 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:41:47 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=84=EF=B8=8F=20perf:=20Cut=20Serial=20?= =?UTF-8?q?and=20Redundant=20Database=20Reads=20(#16711)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: Cut Serial and Redundant Database Reads * perf: Pick the Earliest Balance per User in MongoDB --- api/server/controllers/agents/v1.js | 6 ++-- api/server/routes/__tests__/convos.spec.js | 2 ++ api/server/routes/convos.js | 4 ++- config/list-balances.js | 11 ++++++-- config/user-stats.js | 28 ++++++++++--------- .../data-schemas/src/methods/agentApiKey.ts | 6 +++- .../data-schemas/src/methods/agentCategory.ts | 10 ++++--- .../src/methods/conversationTag.ts | 16 +++++------ .../src/methods/triggerDelivery.ts | 11 ++++++-- 9 files changed, 59 insertions(+), 35 deletions(-) diff --git a/api/server/controllers/agents/v1.js b/api/server/controllers/agents/v1.js index 3fe8db05f5..e2a246529c 100644 --- a/api/server/controllers/agents/v1.js +++ b/api/server/controllers/agents/v1.js @@ -2129,8 +2129,10 @@ const revertAgentVersionHandler = async (req, res) => { */ const getAgentCategories = async (_req, res) => { try { - const categories = await db.getCategoriesWithCounts(); - const promotedCount = await db.countPromotedAgents(); + const [categories, promotedCount] = await Promise.all([ + db.getCategoriesWithCounts(), + db.countPromotedAgents(), + ]); const formattedCategories = categories.map((category) => ({ value: category.value, label: category.label, diff --git a/api/server/routes/__tests__/convos.spec.js b/api/server/routes/__tests__/convos.spec.js index 1fd874b4c6..ebaf67a0d5 100644 --- a/api/server/routes/__tests__/convos.spec.js +++ b/api/server/routes/__tests__/convos.spec.js @@ -2090,6 +2090,7 @@ describe('Convos Routes', () => { context: `POST /api/convos/archive ${mockConversationId}`, preserveUpdatedAt: true, noUpsert: true, + appendMessageIds: [], }, ); }); @@ -2123,6 +2124,7 @@ describe('Convos Routes', () => { context: `POST /api/convos/archive ${mockConversationId}`, preserveUpdatedAt: true, noUpsert: true, + appendMessageIds: [], }, ); }); diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index f901e48dbf..7efc784d00 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -661,6 +661,8 @@ router.post('/archive', validateConvoAccess, async (req, res) => { preserveUpdatedAt: true, /** Without timestamps, an upsert would insert a conversation that has none. */ noUpsert: true, + /** Metadata-only: skip rebuilding `messages` so a concurrent append is not erased. */ + appendMessageIds: [], }, ); @@ -759,7 +761,7 @@ router.post('/update', validateConvoAccess, configMiddleware, async (req, res) = interfaceConfig: req?.config?.interfaceConfig, }, { conversationId, title: sanitizedTitle }, - { context: `POST /api/convos/update ${conversationId}` }, + { context: `POST /api/convos/update ${conversationId}`, appendMessageIds: [] }, ); res.status(201).json(dbResponse); } catch (error) { diff --git a/config/list-balances.js b/config/list-balances.js index fd494308e8..46d32dd5d5 100644 --- a/config/list-balances.js +++ b/config/list-balances.js @@ -15,9 +15,16 @@ const connect = require('./connect'); console.purple('Show the balance of all users'); console.purple('-----------------------------'); - let users = await User.find({}); + const users = await User.find({}); + /** Earliest balance per user, matching the previous per-user `findOne().sort({ _id: 1 })` */ + const balances = await Balance.aggregate([ + { $match: { user: { $in: users.map((user) => user._id) } } }, + { $sort: { _id: 1 } }, + { $group: { _id: '$user', tokenCredits: { $first: '$tokenCredits' } } }, + ]); + const balanceByUser = new Map(balances.map((balance) => [String(balance._id), balance])); for (const user of users) { - let balance = await Balance.findOne({ user: user._id }).sort({ _id: 1 }); + const balance = balanceByUser.get(String(user._id)) ?? null; if (balance !== null) { console.green(`User ${user.name} (${user.email}) has a balance of ${balance.tokenCredits}`); } else { diff --git a/config/user-stats.js b/config/user-stats.js index b9688e6659..d446694fda 100644 --- a/config/user-stats.js +++ b/config/user-stats.js @@ -15,19 +15,21 @@ const connect = require('./connect'); console.purple('Show the stats of all users'); console.purple('-----------------------------'); - let users = await User.find({}); - let userData = []; - for (const user of users) { - let conversationsCount = (await Conversation.countDocuments({ user: user._id })) ?? 0; - let messagesCount = (await Message.countDocuments({ user: user._id })) ?? 0; - - userData.push({ - User: user.name, - Email: user.email, - Conversations: conversationsCount, - Messages: messagesCount, - }); - } + const countByUser = async (Model) => { + const rows = await Model.aggregate([{ $group: { _id: '$user', count: { $sum: 1 } } }]); + return new Map(rows.map(({ _id, count }) => [String(_id), count])); + }; + const [users, conversationCounts, messageCounts] = await Promise.all([ + User.find({}), + countByUser(Conversation), + countByUser(Message), + ]); + const userData = users.map((user) => ({ + User: user.name, + Email: user.email, + Conversations: conversationCounts.get(String(user._id)) ?? 0, + Messages: messageCounts.get(String(user._id)) ?? 0, + })); userData.sort((a, b) => { if (a.Conversations !== b.Conversations) { diff --git a/packages/data-schemas/src/methods/agentApiKey.ts b/packages/data-schemas/src/methods/agentApiKey.ts index c3de31f6bc..a21edc434a 100644 --- a/packages/data-schemas/src/methods/agentApiKey.ts +++ b/packages/data-schemas/src/methods/agentApiKey.ts @@ -79,7 +79,11 @@ export function createAgentApiKeyMethods(mongoose: typeof import('mongoose')): { return null; } - await AgentApiKey.updateOne({ _id: keyDoc._id }, { $set: { lastUsedAt: new Date() } }); + AgentApiKey.updateOne({ _id: keyDoc._id }, { $set: { lastUsedAt: new Date() } }).catch( + (error: unknown) => { + logger.error('[validateAgentApiKey] Error updating lastUsedAt:', error); + }, + ); return { userId: keyDoc.userId, diff --git a/packages/data-schemas/src/methods/agentCategory.ts b/packages/data-schemas/src/methods/agentCategory.ts index cb26923e23..751d4df1f7 100644 --- a/packages/data-schemas/src/methods/agentCategory.ts +++ b/packages/data-schemas/src/methods/agentCategory.ts @@ -44,13 +44,15 @@ export function createAgentCategoryMethods(mongoose: typeof import('mongoose')): async function getCategoriesWithCounts(): Promise<(IAgentCategory & { agentCount: number })[]> { const Agent = mongoose.models.Agent; - const categoryCounts = await Agent.aggregate([ - { $match: { category: { $exists: true, $ne: null } } }, - { $group: { _id: '$category', count: { $sum: 1 } } }, + const [categoryCounts, categories] = await Promise.all([ + Agent.aggregate([ + { $match: { category: { $exists: true, $ne: null } } }, + { $group: { _id: '$category', count: { $sum: 1 } } }, + ]), + getActiveCategories(), ]); const countMap = new Map(categoryCounts.map((c) => [c._id, c.count])); - const categories = await getActiveCategories(); return categories.map((category) => ({ ...category, diff --git a/packages/data-schemas/src/methods/conversationTag.ts b/packages/data-schemas/src/methods/conversationTag.ts index cc24070b9a..5267f8eae5 100644 --- a/packages/data-schemas/src/methods/conversationTag.ts +++ b/packages/data-schemas/src/methods/conversationTag.ts @@ -411,7 +411,7 @@ export function createConversationTagMethods(mongoose: typeof import('mongoose') const ConversationTag = mongoose.models.ConversationTag as Model; const Conversation = mongoose.models.Conversation; - const conversation = await Conversation.findOne({ user, conversationId }).lean(); + const conversation = await Conversation.findOne({ user, conversationId }, 'tags').lean(); if (!conversation) { throw new Error('Conversation not found'); } @@ -455,15 +455,13 @@ export function createConversationTagMethods(mongoose: typeof import('mongoose') await tenantSafeBulkWrite(ConversationTag, bulkOps); } - const updatedConversation = ( - await Conversation.findOneAndUpdate( - { user, conversationId }, - { $set: { tags: [...newTags] } }, - { new: true }, - ) - ).toObject(); + const updatedConversation = await Conversation.findOneAndUpdate( + { user, conversationId }, + { $set: { tags: [...newTags] } }, + { new: true, projection: { tags: 1 } }, + ).lean<{ tags: string[] }>(); - return updatedConversation.tags; + return (updatedConversation as { tags: string[] }).tags; } catch (error) { logger.error('[updateTagsForConversation] Error updating tags', error); throw new Error('Error updating tags for conversation'); diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts index 22cab0aded..3f9ce26434 100644 --- a/packages/data-schemas/src/methods/triggerDelivery.ts +++ b/packages/data-schemas/src/methods/triggerDelivery.ts @@ -4856,10 +4856,15 @@ export function createAgentTriggerDeliveryMethods( .lean(); if (activity != null && markers.length > 0) activity.found = true; let recovered = 0; + const users = + markers.length > 0 + ? await mongoose.models.User.find({ _id: { $in: markers.map((marker) => marker._id) } }) + .select('agentTriggerDeletionStartedAt') + .lean>() + : []; + const usersById = new Map(users.map((found) => [String(found._id), found])); for (const marker of markers) { - const user = await mongoose.models.User.findById(marker._id) - .select('agentTriggerDeletionStartedAt') - .lean<{ agentTriggerDeletionStartedAt?: Date }>(); + const user = usersById.get(String(marker._id)) ?? null; if (user != null) { if (user.agentTriggerDeletionStartedAt?.getTime() === marker.fenceStartedAt.getTime()) { // Move a still-owned pre-commit marker behind this bounded scan so