mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
🗄️ perf: Cut Serial and Redundant Database Reads (#16711)
* perf: Cut Serial and Redundant Database Reads * perf: Pick the Earliest Balance per User in MongoDB
This commit is contained in:
parent
b24226672d
commit
6f4e36665e
9 changed files with 59 additions and 35 deletions
|
|
@ -2129,8 +2129,10 @@ const revertAgentVersionHandler = async (req, res) => {
|
|||
*/
|
||||
const getAgentCategories = async (_req, res) => {
|
||||
try {
|
||||
const categories = await db.getCategoriesWithCounts();
|
||||
const promotedCount = await db.countPromotedAgents();
|
||||
const [categories, promotedCount] = await Promise.all([
|
||||
db.getCategoriesWithCounts(),
|
||||
db.countPromotedAgents(),
|
||||
]);
|
||||
const formattedCategories = categories.map((category) => ({
|
||||
value: category.value,
|
||||
label: category.label,
|
||||
|
|
|
|||
|
|
@ -2090,6 +2090,7 @@ describe('Convos Routes', () => {
|
|||
context: `POST /api/convos/archive ${mockConversationId}`,
|
||||
preserveUpdatedAt: true,
|
||||
noUpsert: true,
|
||||
appendMessageIds: [],
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -2123,6 +2124,7 @@ describe('Convos Routes', () => {
|
|||
context: `POST /api/convos/archive ${mockConversationId}`,
|
||||
preserveUpdatedAt: true,
|
||||
noUpsert: true,
|
||||
appendMessageIds: [],
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -661,6 +661,8 @@ router.post('/archive', validateConvoAccess, async (req, res) => {
|
|||
preserveUpdatedAt: true,
|
||||
/** Without timestamps, an upsert would insert a conversation that has none. */
|
||||
noUpsert: true,
|
||||
/** Metadata-only: skip rebuilding `messages` so a concurrent append is not erased. */
|
||||
appendMessageIds: [],
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -759,7 +761,7 @@ router.post('/update', validateConvoAccess, configMiddleware, async (req, res) =
|
|||
interfaceConfig: req?.config?.interfaceConfig,
|
||||
},
|
||||
{ conversationId, title: sanitizedTitle },
|
||||
{ context: `POST /api/convos/update ${conversationId}` },
|
||||
{ context: `POST /api/convos/update ${conversationId}`, appendMessageIds: [] },
|
||||
);
|
||||
res.status(201).json(dbResponse);
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -15,9 +15,16 @@ const connect = require('./connect');
|
|||
console.purple('Show the balance of all users');
|
||||
console.purple('-----------------------------');
|
||||
|
||||
let users = await User.find({});
|
||||
const users = await User.find({});
|
||||
/** Earliest balance per user, matching the previous per-user `findOne().sort({ _id: 1 })` */
|
||||
const balances = await Balance.aggregate([
|
||||
{ $match: { user: { $in: users.map((user) => user._id) } } },
|
||||
{ $sort: { _id: 1 } },
|
||||
{ $group: { _id: '$user', tokenCredits: { $first: '$tokenCredits' } } },
|
||||
]);
|
||||
const balanceByUser = new Map(balances.map((balance) => [String(balance._id), balance]));
|
||||
for (const user of users) {
|
||||
let balance = await Balance.findOne({ user: user._id }).sort({ _id: 1 });
|
||||
const balance = balanceByUser.get(String(user._id)) ?? null;
|
||||
if (balance !== null) {
|
||||
console.green(`User ${user.name} (${user.email}) has a balance of ${balance.tokenCredits}`);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -15,19 +15,21 @@ const connect = require('./connect');
|
|||
console.purple('Show the stats of all users');
|
||||
console.purple('-----------------------------');
|
||||
|
||||
let users = await User.find({});
|
||||
let userData = [];
|
||||
for (const user of users) {
|
||||
let conversationsCount = (await Conversation.countDocuments({ user: user._id })) ?? 0;
|
||||
let messagesCount = (await Message.countDocuments({ user: user._id })) ?? 0;
|
||||
|
||||
userData.push({
|
||||
User: user.name,
|
||||
Email: user.email,
|
||||
Conversations: conversationsCount,
|
||||
Messages: messagesCount,
|
||||
});
|
||||
}
|
||||
const countByUser = async (Model) => {
|
||||
const rows = await Model.aggregate([{ $group: { _id: '$user', count: { $sum: 1 } } }]);
|
||||
return new Map(rows.map(({ _id, count }) => [String(_id), count]));
|
||||
};
|
||||
const [users, conversationCounts, messageCounts] = await Promise.all([
|
||||
User.find({}),
|
||||
countByUser(Conversation),
|
||||
countByUser(Message),
|
||||
]);
|
||||
const userData = users.map((user) => ({
|
||||
User: user.name,
|
||||
Email: user.email,
|
||||
Conversations: conversationCounts.get(String(user._id)) ?? 0,
|
||||
Messages: messageCounts.get(String(user._id)) ?? 0,
|
||||
}));
|
||||
|
||||
userData.sort((a, b) => {
|
||||
if (a.Conversations !== b.Conversations) {
|
||||
|
|
|
|||
|
|
@ -79,7 +79,11 @@ export function createAgentApiKeyMethods(mongoose: typeof import('mongoose')): {
|
|||
return null;
|
||||
}
|
||||
|
||||
await AgentApiKey.updateOne({ _id: keyDoc._id }, { $set: { lastUsedAt: new Date() } });
|
||||
AgentApiKey.updateOne({ _id: keyDoc._id }, { $set: { lastUsedAt: new Date() } }).catch(
|
||||
(error: unknown) => {
|
||||
logger.error('[validateAgentApiKey] Error updating lastUsedAt:', error);
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
userId: keyDoc.userId,
|
||||
|
|
|
|||
|
|
@ -44,13 +44,15 @@ export function createAgentCategoryMethods(mongoose: typeof import('mongoose')):
|
|||
async function getCategoriesWithCounts(): Promise<(IAgentCategory & { agentCount: number })[]> {
|
||||
const Agent = mongoose.models.Agent;
|
||||
|
||||
const categoryCounts = await Agent.aggregate([
|
||||
{ $match: { category: { $exists: true, $ne: null } } },
|
||||
{ $group: { _id: '$category', count: { $sum: 1 } } },
|
||||
const [categoryCounts, categories] = await Promise.all([
|
||||
Agent.aggregate([
|
||||
{ $match: { category: { $exists: true, $ne: null } } },
|
||||
{ $group: { _id: '$category', count: { $sum: 1 } } },
|
||||
]),
|
||||
getActiveCategories(),
|
||||
]);
|
||||
|
||||
const countMap = new Map(categoryCounts.map((c) => [c._id, c.count]));
|
||||
const categories = await getActiveCategories();
|
||||
|
||||
return categories.map((category) => ({
|
||||
...category,
|
||||
|
|
|
|||
|
|
@ -411,7 +411,7 @@ export function createConversationTagMethods(mongoose: typeof import('mongoose')
|
|||
const ConversationTag = mongoose.models.ConversationTag as Model<IConversationTag>;
|
||||
const Conversation = mongoose.models.Conversation;
|
||||
|
||||
const conversation = await Conversation.findOne({ user, conversationId }).lean();
|
||||
const conversation = await Conversation.findOne({ user, conversationId }, 'tags').lean();
|
||||
if (!conversation) {
|
||||
throw new Error('Conversation not found');
|
||||
}
|
||||
|
|
@ -455,15 +455,13 @@ export function createConversationTagMethods(mongoose: typeof import('mongoose')
|
|||
await tenantSafeBulkWrite(ConversationTag, bulkOps);
|
||||
}
|
||||
|
||||
const updatedConversation = (
|
||||
await Conversation.findOneAndUpdate(
|
||||
{ user, conversationId },
|
||||
{ $set: { tags: [...newTags] } },
|
||||
{ new: true },
|
||||
)
|
||||
).toObject();
|
||||
const updatedConversation = await Conversation.findOneAndUpdate(
|
||||
{ user, conversationId },
|
||||
{ $set: { tags: [...newTags] } },
|
||||
{ new: true, projection: { tags: 1 } },
|
||||
).lean<{ tags: string[] }>();
|
||||
|
||||
return updatedConversation.tags;
|
||||
return (updatedConversation as { tags: string[] }).tags;
|
||||
} catch (error) {
|
||||
logger.error('[updateTagsForConversation] Error updating tags', error);
|
||||
throw new Error('Error updating tags for conversation');
|
||||
|
|
|
|||
|
|
@ -4856,10 +4856,15 @@ export function createAgentTriggerDeliveryMethods(
|
|||
.lean<IAgentTriggerUserPurge[]>();
|
||||
if (activity != null && markers.length > 0) activity.found = true;
|
||||
let recovered = 0;
|
||||
const users =
|
||||
markers.length > 0
|
||||
? await mongoose.models.User.find({ _id: { $in: markers.map((marker) => marker._id) } })
|
||||
.select('agentTriggerDeletionStartedAt')
|
||||
.lean<Array<{ _id: Types.ObjectId; agentTriggerDeletionStartedAt?: Date }>>()
|
||||
: [];
|
||||
const usersById = new Map(users.map((found) => [String(found._id), found]));
|
||||
for (const marker of markers) {
|
||||
const user = await mongoose.models.User.findById(marker._id)
|
||||
.select('agentTriggerDeletionStartedAt')
|
||||
.lean<{ agentTriggerDeletionStartedAt?: Date }>();
|
||||
const user = usersById.get(String(marker._id)) ?? null;
|
||||
if (user != null) {
|
||||
if (user.agentTriggerDeletionStartedAt?.getTime() === marker.fenceStartedAt.getTime()) {
|
||||
// Move a still-owned pre-commit marker behind this bounded scan so
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue