CERBERUS/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-08-28 04:37:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

🧵 fix: Close Child-Thread Read and Search-Cleanup Gaps (#15055)

Browse source 
* fix: close child thread read and cleanup gaps

* fix: preserve child search cleanup invariants

* test: complete mocked update result

* perf: parallelize scoped message reads

* fix: close child thread compatibility gaps

* test: expect preserved cleanup failure

* fix: reconcile legacy Meili cleanup markers
This commit is contained in:
Danny Avila 2026-08-21 00:44:37 -04:00 • committed by GitHub
parent c7e355b219
commit d0f9d5625e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 732 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

93
api/db/indexSync.js
View file

@ -230,44 +230,54 @@ async function performSync(flowManager, flowId, flowType) {
await batchResetMeiliFlags(Conversation.collection);
}
// Check if we need to sync messages
logger.info('[indexSync] Requesting message sync progress...');
const messageProgress = await Message.getSyncProgress();
if (!messageProgress.isComplete || settingsUpdated) {
logger.info(
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
);
const messageCount = messageProgress.totalDocuments;
const messagesIndexed = messageProgress.totalProcessed;
const unindexedMessages = messageCount - messagesIndexed;
const messagesPendingCleanup = messageProgress.pendingCleanup ?? 0;
const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
if (
settingsUpdated ||
noneIndexed ||
unindexedMessages > syncThreshold ||
messagesPendingCleanup > 0
) {
if (noneIndexed && !settingsUpdated) {
logger.info('[indexSync] No messages marked as indexed, forcing full sync');
}
let messageSyncError;
try {
// Check if we need to sync messages
logger.info('[indexSync] Requesting message sync progress...');
const messageProgress = await Message.getSyncProgress();
if (!messageProgress.isComplete || settingsUpdated) {
logger.info(
messagesPendingCleanup > 0
? `[indexSync] Starting message sync (${unindexedMessages} unindexed, ${messagesPendingCleanup} pending cleanup)`
: `[indexSync] Starting message sync (${unindexedMessages} unindexed)`,
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
);
await Message.syncWithMeili();
messagesSync = true;
} else if (unindexedMessages > 0) {
const messageCount = messageProgress.totalDocuments;
const messagesIndexed = messageProgress.totalProcessed;
const unindexedMessages = messageCount - messagesIndexed;
const messagesPendingCleanup = messageProgress.pendingCleanup ?? 0;
const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
if (settingsUpdated || noneIndexed || unindexedMessages > syncThreshold) {
if (noneIndexed && !settingsUpdated) {
logger.info('[indexSync] No messages marked as indexed, forcing full sync');
}
logger.info(
messagesPendingCleanup > 0
? `[indexSync] Starting message sync (${unindexedMessages} unindexed, ${messagesPendingCleanup} pending cleanup)`
: `[indexSync] Starting message sync (${unindexedMessages} unindexed)`,
);
await Message.syncWithMeili();
messagesSync = true;
} else if (messagesPendingCleanup > 0) {
logger.info(
`[indexSync] Cleaning ${messagesPendingCleanup} excluded messages from search`,
);
await Message.cleanupExcludedMeiliIndex();
messagesSync = true;
} else if (unindexedMessages > 0) {
logger.info(
`[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
);
}
} else {
logger.info(
`[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
`[indexSync] Messages are fully synced: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments}`,
);
}
} else {
logger.info(
`[indexSync] Messages are fully synced: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments}`,
} catch (error) {
messageSyncError = error;
logger.error(
'[indexSync] Message reconciliation failed; continuing with conversations:',
error,
);
}
@ -284,12 +294,7 @@ async function performSync(flowManager, flowId, flowType) {
const convosPendingCleanup = convoProgress.pendingCleanup ?? 0;
const noneConvosIndexed = convosIndexed === 0 && unindexedConvos > 0;
if (
settingsUpdated ||
noneConvosIndexed ||
unindexedConvos > syncThreshold ||
convosPendingCleanup > 0
) {
if (settingsUpdated || noneConvosIndexed || unindexedConvos > syncThreshold) {
if (noneConvosIndexed && !settingsUpdated) {
logger.info('[indexSync] No conversations marked as indexed, forcing full sync');
}
@ -300,6 +305,12 @@ async function performSync(flowManager, flowId, flowType) {
);
await Conversation.syncWithMeili();
convosSync = true;
} else if (convosPendingCleanup > 0) {
logger.info(
`[indexSync] Cleaning ${convosPendingCleanup} excluded conversations from search`,
);
await Conversation.cleanupExcludedMeiliIndex();
convosSync = true;
} else if (unindexedConvos > 0) {
logger.info(
`[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`,
@ -311,6 +322,10 @@ async function performSync(flowManager, flowId, flowType) {
);
}
if (messageSyncError) {
throw messageSyncError;
}
return { messagesSync, convosSync };
} finally {
if (indexingDisabled === true) {

58
api/db/indexSync.spec.js
View file

@ -25,6 +25,7 @@ const createMockModel = (collectionName) => ({
collection: { name: collectionName },
getSyncProgress: jest.fn(),
syncWithMeili: jest.fn(),
cleanupExcludedMeiliIndex: jest.fn(),
countDocuments: jest.fn(),
});
@ -528,7 +529,7 @@ describe('performSync() - syncThreshold logic', () => {
);
});
test('forces sync when search contains documents that are now excluded', async () => {
test('runs bounded cleanup when search contains documents that are now excluded', async () => {
Message.getSyncProgress.mockResolvedValue({
totalProcessed: 100,
totalDocuments: 100,
@ -545,10 +546,61 @@ describe('performSync() - syncThreshold logic', () => {
const indexSync = require('./indexSync');
await indexSync();
expect(Message.syncWithMeili).toHaveBeenCalledTimes(1);
expect(Message.syncWithMeili).not.toHaveBeenCalled();
expect(Message.cleanupExcludedMeiliIndex).toHaveBeenCalledTimes(1);
expect(Conversation.syncWithMeili).not.toHaveBeenCalled();
expect(mockLogger.info).toHaveBeenCalledWith(
'[indexSync] Starting message sync (0 unindexed, 1 pending cleanup)',
'[indexSync] Cleaning 1 excluded messages from search',
);
});
test('does not start cleanup for excluded documents that were never indexed', async () => {
Message.getSyncProgress.mockResolvedValue({
totalProcessed: 100,
totalDocuments: 100,
pendingCleanup: 0,
isComplete: true,
});
Conversation.getSyncProgress.mockResolvedValue({
totalProcessed: 50,
totalDocuments: 50,
pendingCleanup: 0,
isComplete: true,
});
const indexSync = require('./indexSync');
await indexSync();
expect(Message.syncWithMeili).not.toHaveBeenCalled();
expect(Message.cleanupExcludedMeiliIndex).not.toHaveBeenCalled();
expect(Conversation.syncWithMeili).not.toHaveBeenCalled();
expect(Conversation.cleanupExcludedMeiliIndex).not.toHaveBeenCalled();
});
test('continues conversation cleanup when message cleanup fails transiently', async () => {
const cleanupError = new Error('message cleanup timed out');
Message.getSyncProgress.mockResolvedValue({
totalProcessed: 100,
totalDocuments: 100,
pendingCleanup: 1,
isComplete: false,
});
Message.cleanupExcludedMeiliIndex.mockRejectedValue(cleanupError);
Conversation.getSyncProgress.mockResolvedValue({
totalProcessed: 50,
totalDocuments: 50,
pendingCleanup: 1,
isComplete: false,
});
const indexSync = require('./indexSync');
await expect(indexSync()).rejects.toThrow(cleanupError);
expect(Message.cleanupExcludedMeiliIndex).toHaveBeenCalledTimes(1);
expect(Conversation.cleanupExcludedMeiliIndex).toHaveBeenCalledTimes(1);
expect(mockLogger.error).toHaveBeenCalledWith(
'[indexSync] Message reconciliation failed; continuing with conversations:',
cleanupError,
);
});
});