mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
* 🪪 fix: Adopt MCP OAuth Callback Tokens Without Re-Prompting or Losing the Lease A chat request loading OAuth tokens joins the `mcp_get_tokens` flow a catalog discovery left behind and polls it. The OAuth callback deletes that entry when it stores the new tokens, so the poll ends with "Flow state not found", the factory reports the tokens missing, drops the `mcp_oauth` flow the callback just completed, and starts a second interactive flow. Once tokens arrive, the connection leases the publication generation it captured before the wait; the callback's publication moved it, so the connection is torn down as it connects and the agent fails with AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE. `createFlowWithHandler` now serves a completed result at once, monitors only pending attempts and retained failures, and replaces a failure nobody retains; a waiter whose flow disappears gets `FlowStateNotFoundError`. The factory treats that as a credential change: it re-reads storage once, after letting the connection build re-capture its generation. The authorization transaction and a silent refresh attach the generation they publish to the tokens they release, and a build adopts that generation for credentials obtained after its capture. * 🪪 fix: Claim Flow Attempts Atomically and Lease Under the Stored Generation A connection now adopts a carried publication generation only while it is the generation currently stored, instead of comparing the token's exchange time to its capture, so a callback that exchanged before the build began and published after it is still adopted and a retired generation still leaves the fence in place. Installing a flow attempt is an atomic compare-and-set against the attempt the caller observed, on Redis and on the default in-memory store, so concurrent replacements of a failed attempt, or concurrent creations of an absent one, run one handler while the rest join it. An aborted caller is rejected before a cached result is served, and a joiner that aborts leaves the shared flow to its owner and the other waiters. * 🪪 fix: Return Generation-Tagged Callback Tokens and Route Flow Claims Through the Cluster Executor The authorization transaction now returns the same generation-tagged tokens it releases to the OAuth completion and the pending token waiters, so the callback route's tool-flow completion carries the published generation to the connection waiting on it. The Action OAuth and index-sync flow managers pass the cluster-aware Redis script executor, so their flow claims and guarded settlements route through slot-master selection and READONLY recovery instead of the raw client.
431 lines
15 KiB
JavaScript
431 lines
15 KiB
JavaScript
const mongoose = require('mongoose');
|
|
const { MeiliSearch } = require('meilisearch');
|
|
const { logger } = require('@librechat/data-schemas');
|
|
const { CacheKeys } = require('librechat-data-provider');
|
|
const { isEnabled, FlowStateManager, evalKeyvRedisScript } = require('@librechat/api');
|
|
const { getLogStores } = require('~/cache');
|
|
const { batchResetMeiliFlags } = require('./utils');
|
|
|
|
const searchEnabled = isEnabled(process.env.SEARCH);
|
|
const indexingDisabled = isEnabled(process.env.MEILI_NO_SYNC);
|
|
let currentTimeout = null;
|
|
|
|
const defaultSyncThreshold = 1000;
|
|
const syncThreshold = process.env.MEILI_SYNC_THRESHOLD
|
|
? parseInt(process.env.MEILI_SYNC_THRESHOLD, 10)
|
|
: defaultSyncThreshold;
|
|
|
|
class MeiliSearchClient {
|
|
static instance = null;
|
|
|
|
static getInstance() {
|
|
if (!MeiliSearchClient.instance) {
|
|
if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY) {
|
|
throw new Error('Meilisearch configuration is missing.');
|
|
}
|
|
MeiliSearchClient.instance = new MeiliSearch({
|
|
host: process.env.MEILI_HOST,
|
|
apiKey: process.env.MEILI_MASTER_KEY,
|
|
});
|
|
}
|
|
return MeiliSearchClient.instance;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Deletes documents from MeiliSearch index that are missing the user field
|
|
* @param {import('meilisearch').Index} index - MeiliSearch index instance
|
|
* @param {string} indexName - Name of the index for logging
|
|
* @returns {Promise<number>} - Number of documents deleted
|
|
*/
|
|
async function deleteDocumentsWithoutUserField(index, indexName) {
|
|
let deletedCount = 0;
|
|
let offset = 0;
|
|
const batchSize = 1000;
|
|
|
|
try {
|
|
while (true) {
|
|
const searchResult = await index.search('', {
|
|
limit: batchSize,
|
|
offset: offset,
|
|
});
|
|
|
|
if (searchResult.hits.length === 0) {
|
|
break;
|
|
}
|
|
|
|
const idsToDelete = searchResult.hits.filter((hit) => !hit.user).map((hit) => hit.id);
|
|
|
|
if (idsToDelete.length > 0) {
|
|
logger.info(
|
|
`[indexSync] Deleting ${idsToDelete.length} documents without user field from ${indexName} index`,
|
|
);
|
|
await index.deleteDocuments(idsToDelete);
|
|
deletedCount += idsToDelete.length;
|
|
}
|
|
|
|
if (searchResult.hits.length < batchSize) {
|
|
break;
|
|
}
|
|
|
|
offset += batchSize;
|
|
}
|
|
|
|
if (deletedCount > 0) {
|
|
logger.info(`[indexSync] Deleted ${deletedCount} orphaned documents from ${indexName} index`);
|
|
}
|
|
} catch (error) {
|
|
logger.error(`[indexSync] Error deleting documents from ${indexName}:`, error);
|
|
}
|
|
|
|
return deletedCount;
|
|
}
|
|
|
|
/**
|
|
* Ensures indexes have proper filterable attributes configured and checks if documents have user field
|
|
* @param {MeiliSearch} client - MeiliSearch client instance
|
|
* @returns {Promise<{settingsUpdated: boolean, orphanedDocsFound: boolean}>} - Status of what was done
|
|
*/
|
|
async function ensureFilterableAttributes(client) {
|
|
let settingsUpdated = false;
|
|
let hasOrphanedDocs = false;
|
|
|
|
try {
|
|
// Check and update messages index
|
|
try {
|
|
const messagesIndex = client.index('messages');
|
|
const settings = await messagesIndex.getSettings();
|
|
|
|
if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
|
|
logger.info('[indexSync] Configuring messages index to filter by user...');
|
|
await messagesIndex.updateSettings({
|
|
filterableAttributes: ['user'],
|
|
});
|
|
logger.info('[indexSync] Messages index configured for user filtering');
|
|
settingsUpdated = true;
|
|
}
|
|
|
|
// Check if existing documents have user field indexed
|
|
try {
|
|
const searchResult = await messagesIndex.search('', { limit: 1 });
|
|
if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
|
|
logger.info(
|
|
'[indexSync] Existing messages missing user field, will clean up orphaned documents...',
|
|
);
|
|
hasOrphanedDocs = true;
|
|
}
|
|
} catch (searchError) {
|
|
logger.debug('[indexSync] Could not check message documents:', searchError.message);
|
|
}
|
|
} catch (error) {
|
|
if (error.code !== 'index_not_found') {
|
|
logger.warn('[indexSync] Could not check/update messages index settings:', error.message);
|
|
}
|
|
}
|
|
|
|
// Check and update conversations index
|
|
try {
|
|
const convosIndex = client.index('convos');
|
|
const settings = await convosIndex.getSettings();
|
|
|
|
if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
|
|
logger.info('[indexSync] Configuring convos index to filter by user...');
|
|
await convosIndex.updateSettings({
|
|
filterableAttributes: ['user'],
|
|
});
|
|
logger.info('[indexSync] Convos index configured for user filtering');
|
|
settingsUpdated = true;
|
|
}
|
|
|
|
// Check if existing documents have user field indexed
|
|
try {
|
|
const searchResult = await convosIndex.search('', { limit: 1 });
|
|
if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
|
|
logger.info(
|
|
'[indexSync] Existing conversations missing user field, will clean up orphaned documents...',
|
|
);
|
|
hasOrphanedDocs = true;
|
|
}
|
|
} catch (searchError) {
|
|
logger.debug('[indexSync] Could not check conversation documents:', searchError.message);
|
|
}
|
|
} catch (error) {
|
|
if (error.code !== 'index_not_found') {
|
|
logger.warn('[indexSync] Could not check/update convos index settings:', error.message);
|
|
}
|
|
}
|
|
|
|
// If either index has orphaned documents, clean them up (but don't force resync)
|
|
if (hasOrphanedDocs) {
|
|
try {
|
|
const messagesIndex = client.index('messages');
|
|
await deleteDocumentsWithoutUserField(messagesIndex, 'messages');
|
|
} catch (error) {
|
|
logger.debug('[indexSync] Could not clean up messages:', error.message);
|
|
}
|
|
|
|
try {
|
|
const convosIndex = client.index('convos');
|
|
await deleteDocumentsWithoutUserField(convosIndex, 'convos');
|
|
} catch (error) {
|
|
logger.debug('[indexSync] Could not clean up convos:', error.message);
|
|
}
|
|
|
|
logger.info('[indexSync] Orphaned documents cleaned up without forcing resync.');
|
|
}
|
|
|
|
if (settingsUpdated) {
|
|
logger.info('[indexSync] Index settings updated. Full re-sync will be triggered.');
|
|
}
|
|
} catch (error) {
|
|
logger.error('[indexSync] Error ensuring filterable attributes:', error);
|
|
}
|
|
|
|
return { settingsUpdated, orphanedDocsFound: hasOrphanedDocs };
|
|
}
|
|
|
|
/**
|
|
* Performs the actual sync operations for messages and conversations
|
|
* @param {FlowStateManager} flowManager - Flow state manager instance
|
|
* @param {string} flowId - Flow identifier
|
|
* @param {string} flowType - Flow type
|
|
*/
|
|
async function performSync(flowManager, flowId, flowType) {
|
|
try {
|
|
if (indexingDisabled === true) {
|
|
logger.info('[indexSync] Indexing is disabled, skipping...');
|
|
return { messagesSync: false, convosSync: false };
|
|
}
|
|
|
|
const Message = mongoose.models.Message;
|
|
const Conversation = mongoose.models.Conversation;
|
|
if (!Message || !Conversation) {
|
|
throw new Error(
|
|
'[indexSync] Models not registered. Ensure createModels() has been called before indexSync.',
|
|
);
|
|
}
|
|
|
|
const client = MeiliSearchClient.getInstance();
|
|
|
|
const { status } = await client.health();
|
|
if (status !== 'available') {
|
|
throw new Error('Meilisearch not available');
|
|
}
|
|
|
|
/** Ensures indexes have proper filterable attributes configured */
|
|
const { settingsUpdated, orphanedDocsFound: _orphanedDocsFound } =
|
|
await ensureFilterableAttributes(client);
|
|
|
|
let messagesSync = false;
|
|
let convosSync = false;
|
|
|
|
// Only reset flags if settings were actually updated (not just for orphaned doc cleanup)
|
|
if (settingsUpdated) {
|
|
logger.info(
|
|
'[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...',
|
|
);
|
|
|
|
// Reset sync flags to force full re-sync
|
|
await batchResetMeiliFlags(Message.collection);
|
|
await batchResetMeiliFlags(Conversation.collection);
|
|
}
|
|
|
|
let messageSyncError;
|
|
try {
|
|
// Check if we need to sync messages
|
|
logger.info('[indexSync] Requesting message sync progress...');
|
|
const messageProgress = await Message.getSyncProgress();
|
|
if (!messageProgress.isComplete || settingsUpdated) {
|
|
logger.info(
|
|
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
|
|
);
|
|
|
|
const messageCount = messageProgress.totalDocuments;
|
|
const messagesIndexed = messageProgress.totalProcessed;
|
|
const unindexedMessages = messageCount - messagesIndexed;
|
|
const messagesPendingIndexing = messageProgress.pendingIndexing ?? 0;
|
|
const messagesPendingCleanup = messageProgress.pendingCleanup ?? 0;
|
|
const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
|
|
|
|
if (
|
|
settingsUpdated ||
|
|
noneIndexed ||
|
|
messagesPendingIndexing > 0 ||
|
|
unindexedMessages > syncThreshold
|
|
) {
|
|
if (noneIndexed && !settingsUpdated) {
|
|
logger.info('[indexSync] No messages marked as indexed, forcing full sync');
|
|
}
|
|
logger.info(
|
|
messagesPendingCleanup > 0
|
|
? `[indexSync] Starting message sync (${unindexedMessages} unindexed, ${messagesPendingCleanup} pending cleanup)`
|
|
: `[indexSync] Starting message sync (${unindexedMessages} unindexed)`,
|
|
);
|
|
await Message.syncWithMeili();
|
|
messagesSync = true;
|
|
} else if (messagesPendingCleanup > 0) {
|
|
logger.info(
|
|
`[indexSync] Cleaning ${messagesPendingCleanup} excluded messages from search`,
|
|
);
|
|
await Message.cleanupExcludedMeiliIndex();
|
|
messagesSync = true;
|
|
} else if (unindexedMessages > 0) {
|
|
logger.info(
|
|
`[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
|
|
);
|
|
}
|
|
} else {
|
|
logger.info(
|
|
`[indexSync] Messages are fully synced: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments}`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
messageSyncError = error;
|
|
logger.error(
|
|
'[indexSync] Message reconciliation failed; continuing with conversations:',
|
|
error,
|
|
);
|
|
}
|
|
|
|
// Check if we need to sync conversations
|
|
const convoProgress = await Conversation.getSyncProgress();
|
|
if (!convoProgress.isComplete || settingsUpdated) {
|
|
logger.info(
|
|
`[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,
|
|
);
|
|
|
|
const convoCount = convoProgress.totalDocuments;
|
|
const convosIndexed = convoProgress.totalProcessed;
|
|
const unindexedConvos = convoCount - convosIndexed;
|
|
const convosPendingIndexing = convoProgress.pendingIndexing ?? 0;
|
|
const convosPendingCleanup = convoProgress.pendingCleanup ?? 0;
|
|
const noneConvosIndexed = convosIndexed === 0 && unindexedConvos > 0;
|
|
|
|
if (
|
|
settingsUpdated ||
|
|
noneConvosIndexed ||
|
|
convosPendingIndexing > 0 ||
|
|
unindexedConvos > syncThreshold
|
|
) {
|
|
if (noneConvosIndexed && !settingsUpdated) {
|
|
logger.info('[indexSync] No conversations marked as indexed, forcing full sync');
|
|
}
|
|
logger.info(
|
|
convosPendingCleanup > 0
|
|
? `[indexSync] Starting convos sync (${unindexedConvos} unindexed, ${convosPendingCleanup} pending cleanup)`
|
|
: `[indexSync] Starting convos sync (${unindexedConvos} unindexed)`,
|
|
);
|
|
await Conversation.syncWithMeili();
|
|
convosSync = true;
|
|
} else if (convosPendingCleanup > 0) {
|
|
logger.info(
|
|
`[indexSync] Cleaning ${convosPendingCleanup} excluded conversations from search`,
|
|
);
|
|
await Conversation.cleanupExcludedMeiliIndex();
|
|
convosSync = true;
|
|
} else if (unindexedConvos > 0) {
|
|
logger.info(
|
|
`[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`,
|
|
);
|
|
}
|
|
} else {
|
|
logger.info(
|
|
`[indexSync] Conversations are fully synced: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments}`,
|
|
);
|
|
}
|
|
|
|
if (messageSyncError) {
|
|
throw messageSyncError;
|
|
}
|
|
|
|
return { messagesSync, convosSync };
|
|
} finally {
|
|
if (indexingDisabled === true) {
|
|
logger.info('[indexSync] Indexing is disabled, skipping cleanup...');
|
|
} else if (flowManager && flowId && flowType) {
|
|
try {
|
|
await flowManager.deleteFlow(flowId, flowType);
|
|
logger.debug('[indexSync] Flow state cleaned up');
|
|
} catch (cleanupErr) {
|
|
logger.debug('[indexSync] Could not clean up flow state:', cleanupErr.message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Main index sync function that uses FlowStateManager to prevent concurrent execution
|
|
*/
|
|
async function indexSync() {
|
|
if (!searchEnabled) {
|
|
return;
|
|
}
|
|
|
|
logger.info('[indexSync] Starting index synchronization check...');
|
|
|
|
// Get or create FlowStateManager instance
|
|
const flowsCache = getLogStores(CacheKeys.FLOWS);
|
|
if (!flowsCache) {
|
|
logger.warn('[indexSync] Flows cache not available, falling back to direct sync');
|
|
return await performSync(null, null, null);
|
|
}
|
|
|
|
const flowManager = new FlowStateManager(flowsCache, {
|
|
ttl: 60000 * 10, // 10 minutes TTL for sync operations
|
|
redisScriptExecutor: evalKeyvRedisScript,
|
|
});
|
|
|
|
// Use a unique flow ID for the sync operation
|
|
const flowId = 'meili-index-sync';
|
|
const flowType = 'MEILI_SYNC';
|
|
|
|
try {
|
|
// This will only execute the handler if no other instance is running the sync
|
|
const result = await flowManager.createFlowWithHandler(flowId, flowType, () =>
|
|
performSync(flowManager, flowId, flowType),
|
|
);
|
|
|
|
if (result.messagesSync || result.convosSync) {
|
|
logger.info('[indexSync] Sync completed successfully');
|
|
} else {
|
|
logger.debug('[indexSync] No sync was needed');
|
|
}
|
|
|
|
return result;
|
|
} catch (err) {
|
|
if (err.message.includes('flow already exists')) {
|
|
logger.info('[indexSync] Sync already running on another instance');
|
|
return;
|
|
}
|
|
|
|
if (err.message.includes('not found')) {
|
|
logger.debug('[indexSync] Creating indices...');
|
|
currentTimeout = setTimeout(async () => {
|
|
try {
|
|
const Message = mongoose.models.Message;
|
|
const Conversation = mongoose.models.Conversation;
|
|
if (!Message || !Conversation) {
|
|
throw new Error(
|
|
'[indexSync] Models not registered. Ensure createModels() has been called before indexSync.',
|
|
);
|
|
}
|
|
await Message.syncWithMeili();
|
|
await Conversation.syncWithMeili();
|
|
} catch (err) {
|
|
logger.error('[indexSync] Trouble creating indices, try restarting the server.', err);
|
|
}
|
|
}, 750);
|
|
} else if (err.message.includes('Meilisearch not configured')) {
|
|
logger.info('[indexSync] Meilisearch not configured, search will be disabled.');
|
|
} else {
|
|
logger.error('[indexSync] error', err);
|
|
}
|
|
}
|
|
}
|
|
|
|
process.on('exit', () => {
|
|
logger.debug('[indexSync] Clearing sync timeouts before exiting...');
|
|
clearTimeout(currentTimeout);
|
|
});
|
|
|
|
module.exports = indexSync;
|