LibreChat/api/db/indexSync.js
Danny Avila 3f2e4777cb
🛬 fix: Land MCP OAuth Callbacks on Waiting Connections (#15876)
* 🪪 fix: Adopt MCP OAuth Callback Tokens Without Re-Prompting or Losing the Lease

A chat request loading OAuth tokens joins the `mcp_get_tokens` flow a catalog
discovery left behind and polls it. The OAuth callback deletes that entry when
it stores the new tokens, so the poll ends with "Flow state not found", the
factory reports the tokens missing, drops the `mcp_oauth` flow the callback
just completed, and starts a second interactive flow. Once tokens arrive, the
connection leases the publication generation it captured before the wait; the
callback's publication moved it, so the connection is torn down as it connects
and the agent fails with AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE.

`createFlowWithHandler` now serves a completed result at once, monitors only
pending attempts and retained failures, and replaces a failure nobody retains;
a waiter whose flow disappears gets `FlowStateNotFoundError`. The factory
treats that as a credential change: it re-reads storage once, after letting
the connection build re-capture its generation. The authorization transaction
and a silent refresh attach the generation they publish to the tokens they
release, and a build adopts that generation for credentials obtained after
its capture.

* 🪪 fix: Claim Flow Attempts Atomically and Lease Under the Stored Generation

A connection now adopts a carried publication generation only while it is the
generation currently stored, instead of comparing the token's exchange time to
its capture, so a callback that exchanged before the build began and published
after it is still adopted and a retired generation still leaves the fence in
place. Installing a flow attempt is an atomic compare-and-set against the
attempt the caller observed, on Redis and on the default in-memory store, so
concurrent replacements of a failed attempt, or concurrent creations of an
absent one, run one handler while the rest join it. An aborted caller is
rejected before a cached result is served, and a joiner that aborts leaves the
shared flow to its owner and the other waiters.

* 🪪 fix: Return Generation-Tagged Callback Tokens and Route Flow Claims Through the Cluster Executor

The authorization transaction now returns the same generation-tagged tokens it
releases to the OAuth completion and the pending token waiters, so the callback
route's tool-flow completion carries the published generation to the connection
waiting on it. The Action OAuth and index-sync flow managers pass the
cluster-aware Redis script executor, so their flow claims and guarded
settlements route through slot-master selection and READONLY recovery instead
of the raw client.
2026-09-13 01:46:41 -04:00

431 lines
15 KiB
JavaScript

const mongoose = require('mongoose');
const { MeiliSearch } = require('meilisearch');
const { logger } = require('@librechat/data-schemas');
const { CacheKeys } = require('librechat-data-provider');
const { isEnabled, FlowStateManager, evalKeyvRedisScript } = require('@librechat/api');
const { getLogStores } = require('~/cache');
const { batchResetMeiliFlags } = require('./utils');
const searchEnabled = isEnabled(process.env.SEARCH);
const indexingDisabled = isEnabled(process.env.MEILI_NO_SYNC);
let currentTimeout = null;
const defaultSyncThreshold = 1000;
const syncThreshold = process.env.MEILI_SYNC_THRESHOLD
? parseInt(process.env.MEILI_SYNC_THRESHOLD, 10)
: defaultSyncThreshold;
class MeiliSearchClient {
static instance = null;
static getInstance() {
if (!MeiliSearchClient.instance) {
if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY) {
throw new Error('Meilisearch configuration is missing.');
}
MeiliSearchClient.instance = new MeiliSearch({
host: process.env.MEILI_HOST,
apiKey: process.env.MEILI_MASTER_KEY,
});
}
return MeiliSearchClient.instance;
}
}
/**
* Deletes documents from MeiliSearch index that are missing the user field
* @param {import('meilisearch').Index} index - MeiliSearch index instance
* @param {string} indexName - Name of the index for logging
* @returns {Promise<number>} - Number of documents deleted
*/
async function deleteDocumentsWithoutUserField(index, indexName) {
let deletedCount = 0;
let offset = 0;
const batchSize = 1000;
try {
while (true) {
const searchResult = await index.search('', {
limit: batchSize,
offset: offset,
});
if (searchResult.hits.length === 0) {
break;
}
const idsToDelete = searchResult.hits.filter((hit) => !hit.user).map((hit) => hit.id);
if (idsToDelete.length > 0) {
logger.info(
`[indexSync] Deleting ${idsToDelete.length} documents without user field from ${indexName} index`,
);
await index.deleteDocuments(idsToDelete);
deletedCount += idsToDelete.length;
}
if (searchResult.hits.length < batchSize) {
break;
}
offset += batchSize;
}
if (deletedCount > 0) {
logger.info(`[indexSync] Deleted ${deletedCount} orphaned documents from ${indexName} index`);
}
} catch (error) {
logger.error(`[indexSync] Error deleting documents from ${indexName}:`, error);
}
return deletedCount;
}
/**
* Ensures indexes have proper filterable attributes configured and checks if documents have user field
* @param {MeiliSearch} client - MeiliSearch client instance
* @returns {Promise<{settingsUpdated: boolean, orphanedDocsFound: boolean}>} - Status of what was done
*/
async function ensureFilterableAttributes(client) {
let settingsUpdated = false;
let hasOrphanedDocs = false;
try {
// Check and update messages index
try {
const messagesIndex = client.index('messages');
const settings = await messagesIndex.getSettings();
if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
logger.info('[indexSync] Configuring messages index to filter by user...');
await messagesIndex.updateSettings({
filterableAttributes: ['user'],
});
logger.info('[indexSync] Messages index configured for user filtering');
settingsUpdated = true;
}
// Check if existing documents have user field indexed
try {
const searchResult = await messagesIndex.search('', { limit: 1 });
if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
logger.info(
'[indexSync] Existing messages missing user field, will clean up orphaned documents...',
);
hasOrphanedDocs = true;
}
} catch (searchError) {
logger.debug('[indexSync] Could not check message documents:', searchError.message);
}
} catch (error) {
if (error.code !== 'index_not_found') {
logger.warn('[indexSync] Could not check/update messages index settings:', error.message);
}
}
// Check and update conversations index
try {
const convosIndex = client.index('convos');
const settings = await convosIndex.getSettings();
if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
logger.info('[indexSync] Configuring convos index to filter by user...');
await convosIndex.updateSettings({
filterableAttributes: ['user'],
});
logger.info('[indexSync] Convos index configured for user filtering');
settingsUpdated = true;
}
// Check if existing documents have user field indexed
try {
const searchResult = await convosIndex.search('', { limit: 1 });
if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
logger.info(
'[indexSync] Existing conversations missing user field, will clean up orphaned documents...',
);
hasOrphanedDocs = true;
}
} catch (searchError) {
logger.debug('[indexSync] Could not check conversation documents:', searchError.message);
}
} catch (error) {
if (error.code !== 'index_not_found') {
logger.warn('[indexSync] Could not check/update convos index settings:', error.message);
}
}
// If either index has orphaned documents, clean them up (but don't force resync)
if (hasOrphanedDocs) {
try {
const messagesIndex = client.index('messages');
await deleteDocumentsWithoutUserField(messagesIndex, 'messages');
} catch (error) {
logger.debug('[indexSync] Could not clean up messages:', error.message);
}
try {
const convosIndex = client.index('convos');
await deleteDocumentsWithoutUserField(convosIndex, 'convos');
} catch (error) {
logger.debug('[indexSync] Could not clean up convos:', error.message);
}
logger.info('[indexSync] Orphaned documents cleaned up without forcing resync.');
}
if (settingsUpdated) {
logger.info('[indexSync] Index settings updated. Full re-sync will be triggered.');
}
} catch (error) {
logger.error('[indexSync] Error ensuring filterable attributes:', error);
}
return { settingsUpdated, orphanedDocsFound: hasOrphanedDocs };
}
/**
* Performs the actual sync operations for messages and conversations
* @param {FlowStateManager} flowManager - Flow state manager instance
* @param {string} flowId - Flow identifier
* @param {string} flowType - Flow type
*/
async function performSync(flowManager, flowId, flowType) {
try {
if (indexingDisabled === true) {
logger.info('[indexSync] Indexing is disabled, skipping...');
return { messagesSync: false, convosSync: false };
}
const Message = mongoose.models.Message;
const Conversation = mongoose.models.Conversation;
if (!Message || !Conversation) {
throw new Error(
'[indexSync] Models not registered. Ensure createModels() has been called before indexSync.',
);
}
const client = MeiliSearchClient.getInstance();
const { status } = await client.health();
if (status !== 'available') {
throw new Error('Meilisearch not available');
}
/** Ensures indexes have proper filterable attributes configured */
const { settingsUpdated, orphanedDocsFound: _orphanedDocsFound } =
await ensureFilterableAttributes(client);
let messagesSync = false;
let convosSync = false;
// Only reset flags if settings were actually updated (not just for orphaned doc cleanup)
if (settingsUpdated) {
logger.info(
'[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...',
);
// Reset sync flags to force full re-sync
await batchResetMeiliFlags(Message.collection);
await batchResetMeiliFlags(Conversation.collection);
}
let messageSyncError;
try {
// Check if we need to sync messages
logger.info('[indexSync] Requesting message sync progress...');
const messageProgress = await Message.getSyncProgress();
if (!messageProgress.isComplete || settingsUpdated) {
logger.info(
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
);
const messageCount = messageProgress.totalDocuments;
const messagesIndexed = messageProgress.totalProcessed;
const unindexedMessages = messageCount - messagesIndexed;
const messagesPendingIndexing = messageProgress.pendingIndexing ?? 0;
const messagesPendingCleanup = messageProgress.pendingCleanup ?? 0;
const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
if (
settingsUpdated ||
noneIndexed ||
messagesPendingIndexing > 0 ||
unindexedMessages > syncThreshold
) {
if (noneIndexed && !settingsUpdated) {
logger.info('[indexSync] No messages marked as indexed, forcing full sync');
}
logger.info(
messagesPendingCleanup > 0
? `[indexSync] Starting message sync (${unindexedMessages} unindexed, ${messagesPendingCleanup} pending cleanup)`
: `[indexSync] Starting message sync (${unindexedMessages} unindexed)`,
);
await Message.syncWithMeili();
messagesSync = true;
} else if (messagesPendingCleanup > 0) {
logger.info(
`[indexSync] Cleaning ${messagesPendingCleanup} excluded messages from search`,
);
await Message.cleanupExcludedMeiliIndex();
messagesSync = true;
} else if (unindexedMessages > 0) {
logger.info(
`[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
);
}
} else {
logger.info(
`[indexSync] Messages are fully synced: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments}`,
);
}
} catch (error) {
messageSyncError = error;
logger.error(
'[indexSync] Message reconciliation failed; continuing with conversations:',
error,
);
}
// Check if we need to sync conversations
const convoProgress = await Conversation.getSyncProgress();
if (!convoProgress.isComplete || settingsUpdated) {
logger.info(
`[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,
);
const convoCount = convoProgress.totalDocuments;
const convosIndexed = convoProgress.totalProcessed;
const unindexedConvos = convoCount - convosIndexed;
const convosPendingIndexing = convoProgress.pendingIndexing ?? 0;
const convosPendingCleanup = convoProgress.pendingCleanup ?? 0;
const noneConvosIndexed = convosIndexed === 0 && unindexedConvos > 0;
if (
settingsUpdated ||
noneConvosIndexed ||
convosPendingIndexing > 0 ||
unindexedConvos > syncThreshold
) {
if (noneConvosIndexed && !settingsUpdated) {
logger.info('[indexSync] No conversations marked as indexed, forcing full sync');
}
logger.info(
convosPendingCleanup > 0
? `[indexSync] Starting convos sync (${unindexedConvos} unindexed, ${convosPendingCleanup} pending cleanup)`
: `[indexSync] Starting convos sync (${unindexedConvos} unindexed)`,
);
await Conversation.syncWithMeili();
convosSync = true;
} else if (convosPendingCleanup > 0) {
logger.info(
`[indexSync] Cleaning ${convosPendingCleanup} excluded conversations from search`,
);
await Conversation.cleanupExcludedMeiliIndex();
convosSync = true;
} else if (unindexedConvos > 0) {
logger.info(
`[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`,
);
}
} else {
logger.info(
`[indexSync] Conversations are fully synced: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments}`,
);
}
if (messageSyncError) {
throw messageSyncError;
}
return { messagesSync, convosSync };
} finally {
if (indexingDisabled === true) {
logger.info('[indexSync] Indexing is disabled, skipping cleanup...');
} else if (flowManager && flowId && flowType) {
try {
await flowManager.deleteFlow(flowId, flowType);
logger.debug('[indexSync] Flow state cleaned up');
} catch (cleanupErr) {
logger.debug('[indexSync] Could not clean up flow state:', cleanupErr.message);
}
}
}
}
/**
* Main index sync function that uses FlowStateManager to prevent concurrent execution
*/
async function indexSync() {
if (!searchEnabled) {
return;
}
logger.info('[indexSync] Starting index synchronization check...');
// Get or create FlowStateManager instance
const flowsCache = getLogStores(CacheKeys.FLOWS);
if (!flowsCache) {
logger.warn('[indexSync] Flows cache not available, falling back to direct sync');
return await performSync(null, null, null);
}
const flowManager = new FlowStateManager(flowsCache, {
ttl: 60000 * 10, // 10 minutes TTL for sync operations
redisScriptExecutor: evalKeyvRedisScript,
});
// Use a unique flow ID for the sync operation
const flowId = 'meili-index-sync';
const flowType = 'MEILI_SYNC';
try {
// This will only execute the handler if no other instance is running the sync
const result = await flowManager.createFlowWithHandler(flowId, flowType, () =>
performSync(flowManager, flowId, flowType),
);
if (result.messagesSync || result.convosSync) {
logger.info('[indexSync] Sync completed successfully');
} else {
logger.debug('[indexSync] No sync was needed');
}
return result;
} catch (err) {
if (err.message.includes('flow already exists')) {
logger.info('[indexSync] Sync already running on another instance');
return;
}
if (err.message.includes('not found')) {
logger.debug('[indexSync] Creating indices...');
currentTimeout = setTimeout(async () => {
try {
const Message = mongoose.models.Message;
const Conversation = mongoose.models.Conversation;
if (!Message || !Conversation) {
throw new Error(
'[indexSync] Models not registered. Ensure createModels() has been called before indexSync.',
);
}
await Message.syncWithMeili();
await Conversation.syncWithMeili();
} catch (err) {
logger.error('[indexSync] Trouble creating indices, try restarting the server.', err);
}
}, 750);
} else if (err.message.includes('Meilisearch not configured')) {
logger.info('[indexSync] Meilisearch not configured, search will be disabled.');
} else {
logger.error('[indexSync] error', err);
}
}
}
process.on('exit', () => {
logger.debug('[indexSync] Clearing sync timeouts before exiting...');
clearTimeout(currentTimeout);
});
module.exports = indexSync;