🛶 chore: Merge Latest Dev Fixes into Canary Sync

This commit is contained in:
Lia 2026-09-28 01:42:05 +00:00
commit c505db2256
55 changed files with 2602 additions and 177 deletions

View file

@ -3172,6 +3172,102 @@ describe('AgentClient - startup telemetry', () => {
errorSpy.mockRestore();
});
it.each([
[
'closed',
'SocketError',
ErrorTypes.MODEL_STREAM_CLOSED,
'stream_closed',
'The model provider closed the connection before the response finished. Try again.',
],
[
'stalled',
'BodyTimeoutError',
ErrorTypes.MODEL_STREAM_STALLED,
'stream_stalled',
'The model provider stopped sending the response, and the request timed out. Try again.',
],
])(
'keeps partial content and a safe %s model error in the agent turn',
async (kind, causeName, type, errorType, prose) => {
jest.clearAllMocks();
const { logger } = require('@librechat/data-schemas');
const { errors: undiciErrors } = require('undici');
const privateValue = 'PRIVATE-TRANSPORT-DIAGNOSTIC';
const cause = new undiciErrors[causeName](privateValue);
const providerError = new TypeError('terminated', { cause });
const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);
mockCreateRun.mockImplementation(async (options) => {
const tracker = options.modelCallbacks.find(
(callback) => callback.name === 'librechat-upstream-model-error-tracker',
);
return {
Graph: null,
processStream: jest.fn(async () => {
tracker.handleLLMError(providerError);
throw new Error('graph failed', { cause: providerError });
}),
getCalibrationRatio: jest.fn(() => 0),
};
});
mockIsHITLEnabled.mockReturnValue(false);
const partial = { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Partial findings' };
const client = new AgentClient({
req: {
user: { id: 'user-123' },
body: {},
config: {
endpoints: { [EModelEndpoint.agents]: {} },
filters: { messages: { pii: {} } },
},
_resumableStreamId: `conversation-stream-${kind}`,
},
res: {},
agent: {
id: 'agent-123',
endpoint: EModelEndpoint.openAI,
provider: EModelEndpoint.openAI,
model_parameters: { model: 'gpt-4' },
hide_sequential_outputs: false,
},
endpointTokenConfig: {},
eventHandlers: {},
contentParts: [partial],
collectedUsage: [],
artifactPromises: [],
});
client.conversationId = `conversation-stream-${kind}`;
client.responseMessageId = `response-stream-${kind}`;
client.parentMessageId = `parent-stream-${kind}`;
client.recordCollectedUsage = jest.fn().mockResolvedValue();
try {
await client.chatCompletion({ payload: [] });
expect(client.contentParts).toEqual(
expect.arrayContaining([
partial,
{
type: ContentTypes.ERROR,
[ContentTypes.ERROR]: `${prose}\n${JSON.stringify({ type })}`,
},
]),
);
expect(JSON.stringify(client.contentParts)).not.toContain(privateValue);
expect(client.recordCollectedUsage).toHaveBeenCalledWith(
expect.objectContaining({ context: 'message' }),
);
expect(errorSpy).toHaveBeenCalledWith(
'[api/server/controllers/agents/client.js #sendCompletion] Upstream model error',
expect.objectContaining({ errorType }),
);
expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(privateValue);
} finally {
errorSpy.mockRestore();
}
},
);
/** A compaction's only record of having been one is the marker on the part it
* produced, and Compact runs on whatever leaf the branch ends with. Without
* the marker on the failure, a compaction that failed on a user leaf keeps a

View file

@ -1088,10 +1088,14 @@ async function readWorkspaceFile({
req,
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
}) {
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
@ -1137,10 +1141,14 @@ async function searchWorkspace({
req,
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
}) {
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
@ -1186,10 +1194,14 @@ async function listWorkspaceFiles({
req,
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
}) {
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
@ -1221,10 +1233,14 @@ async function writeWorkspaceFile({
req,
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
}) {
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
@ -1256,10 +1272,14 @@ async function editWorkspaceFile({
req,
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
}) {
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),
@ -1290,10 +1310,14 @@ async function previewWorkspaceEdit({
req,
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
}) {
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
...(await getCodeApiAuthHeaders(req, bridgeWorkerId)),

View file

@ -2061,6 +2061,8 @@ describe('Code Process', () => {
req: mockReq,
signal: controller.signal,
maxQueueWaitMs: 0,
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
}),
).resolves.toBe(result);
@ -2081,6 +2083,8 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
request: {
protocolVersion: 1,
operation: 'read_file',
@ -2140,6 +2144,8 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
maxRequestTimeoutMs: undefined,
deadlineAtMs: undefined,
request: {
protocolVersion: 1,
operation: 'search_text',
@ -2198,6 +2204,8 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
maxRequestTimeoutMs: undefined,
deadlineAtMs: undefined,
request: {
protocolVersion: 1,
operation: 'list_files',
@ -2258,6 +2266,8 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
maxRequestTimeoutMs: undefined,
deadlineAtMs: undefined,
request: {
protocolVersion: 1,
operation: 'write_file',
@ -2296,11 +2306,15 @@ describe('Code Process', () => {
bridgeWorkerId: 'worker-user-1',
req: mockReq,
expected_base_sha256: 'a'.repeat(64),
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
}),
).resolves.toBe(result);
expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith(
expect.objectContaining({
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
request: {
protocolVersion: 1,
operation: 'edit_file',
@ -2338,11 +2352,15 @@ describe('Code Process', () => {
codeApiBaseUrl: 'https://attached-code.example.com/v1',
executionProfile: 'stateful',
req: mockReq,
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
}),
).resolves.toBe(result);
expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith(
expect.objectContaining({
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
request: {
protocolVersion: 1,
operation: 'preview_edit',

View file

@ -861,6 +861,8 @@ async function reconnectServer({
recoveryPolicy,
mcpApps,
oboIdentityContext,
streamId,
jobCreatedAt,
forceNew: true,
returnOnOAuth: false,
connectionTimeout: Time.THIRTY_SECONDS,
@ -1413,6 +1415,16 @@ function createToolInstance({
);
}
// The schedule service checks the typed cause and verified job identity before
// recording a durable tool failure; other tool errors are a cheap no-op.
await require('~/server/services/Schedules').recordMCPToolAuthFailure({
error,
streamId,
jobCreatedAt,
userId,
serverName,
});
/** Carries the actionable re-auth message; the substring heuristic below would misreport it as an OAuth configuration problem */
if (
error instanceof OpenIDReauthRequiredError ||

View file

@ -92,6 +92,10 @@ jest.mock('./Tools/mcp', () => ({
reinitMCPServer: jest.fn(),
}));
jest.mock('~/server/services/Schedules', () => ({
recordMCPToolAuthFailure: jest.fn(async () => true),
}));
jest.mock('./GraphTokenService', () => ({
getGraphApiToken: jest.fn(),
}));
@ -1660,6 +1664,44 @@ describe('User parameter passing tests', () => {
).resolves.toBe('ordinary output');
});
it('records a typed OBO failure against the scheduled generation before returning an error', async () => {
const user = { id: 'scheduled-owner', role: 'USER' };
const missing = new Error('Unattended provider not configured');
const error = Object.assign(new Error('MCP tool error'), { cause: missing });
const receipt = require('~/server/services/Schedules').recordMCPToolAuthFailure;
require('~/models').getRoleByName.mockResolvedValue({
permissions: { [PermissionTypes.MCP_SERVERS]: { [Permissions.USE]: true } },
});
mockGetMCPManager.mockReturnValue({ callTool: jest.fn().mockRejectedValue(error) });
const tool = await createMCPTool({
user,
toolKey: `test-tool${D}test-server`,
provider: 'openai',
streamId: 'scheduled-conversation',
jobCreatedAt: 42,
config: { type: 'streamable-http', url: 'https://mcp.example.com' },
availableTools: {
[`test-tool${D}test-server`]: {
function: { description: 'Test MCP', parameters: { type: 'object', properties: {} } },
},
},
});
await expect(
tool.func({}, undefined, {
configurable: { user },
metadata: { provider: 'openai', thread_id: 'scheduled-conversation', run_id: 'run-1' },
toolCall: {},
}),
).rejects.toThrow();
expect(receipt).toHaveBeenCalledWith({
error,
streamId: 'scheduled-conversation',
jobCreatedAt: 42,
userId: 'scheduled-owner',
serverName: 'test-server',
});
});
it('keeps shared OAuth recovery alive when one tool caller aborts', async () => {
const mockUser = { id: 'shared-recovery-user', role: 'USER' };
const mockRes = { write: jest.fn(), flush: jest.fn() };
@ -2899,6 +2941,8 @@ describe('User parameter passing tests', () => {
serverName: 'server1',
provider: 'anthropic',
userMCPAuthMap: {},
streamId: 'scheduled-stream',
jobCreatedAt: 42,
});
// Verify all calls to reinitMCPServer had the user
@ -2906,6 +2950,8 @@ describe('User parameter passing tests', () => {
reinitCalls.forEach((call) => {
expect(call.user).toBe(mockUser);
expect(call.user.id).toBe('user-001');
expect(call.streamId).toBe('scheduled-stream');
expect(call.jobCreatedAt).toBe(42);
});
});
@ -2933,12 +2979,16 @@ describe('User parameter passing tests', () => {
provider: 'google',
userMCPAuthMap: {},
availableTools: undefined, // Force reinit
streamId: 'resumed-stream',
jobCreatedAt: 43,
});
// Verify the call to reinitMCPServer had the user
expect(reinitCalls.length).toBe(1);
expect(reinitCalls[0].user).toBe(mockUser);
expect(reinitCalls[0].user.id).toBe('user-002');
expect(reinitCalls[0].streamId).toBe('resumed-stream');
expect(reinitCalls[0].jobCreatedAt).toBe(43);
});
});

View file

@ -84,6 +84,11 @@ module.exports = {
getLimits: invoke('getLimits'),
fireScheduleNow: invoke('fireScheduleNow'),
recordScheduleOutcome: invoke('recordScheduleOutcome'),
recordMCPToolAuthFailure: (input) =>
require('@librechat/api').recordScheduledMCPToolAuthFailure(
input,
() => getService().recordMCPToolAuthFailure,
),
beginScheduledStop: invoke('beginScheduledStop'),
acknowledgeScheduledStopPersistence: invoke('acknowledgeScheduledStopPersistence'),
claimScheduleResume: invoke('claimScheduleResume'),

View file

@ -53,6 +53,7 @@ const {
createRepositoryInstructionLoader,
resolveAttachedWorkspaceCommandTimeoutMax,
resolveAttachedWorkspaceQueueWaitMs,
resolveAttachedWorkspaceRequestTimeoutMs,
createContextProgrammaticBashTool,
resolveCodeExecutionContext,
resolveCodeExecutionWorkspaceContext,
@ -1225,6 +1226,8 @@ async function loadToolDefinitionsWrapper({
const result = await reinitMCPServer({
signal,
user: req.user,
streamId,
jobCreatedAt,
oauthStart,
flowManager,
serverName,
@ -1257,6 +1260,8 @@ async function loadToolDefinitionsWrapper({
const result = await reinitMCPServer({
signal,
user: req.user,
streamId,
jobCreatedAt,
forceNew: true,
oauthStart,
flowManager,
@ -1410,6 +1415,8 @@ async function loadToolDefinitionsWrapper({
const result = await reinitMCPServer({
signal,
user: req.user,
streamId,
jobCreatedAt,
serverName,
configServers,
userMCPAuthMap,
@ -2344,6 +2351,9 @@ async function loadToolsForExecution({
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
codeExecutionContext.codeEnvironmentConfigSchema,
),
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(
codeExecutionContext.codeEnvironmentConfigSchema,
),
})
: createBashExecutionTool({
authHeaders,

View file

@ -10,6 +10,7 @@ const {
isMCPInitializationError,
prepareMCPAuthorizationMutation,
resolveMCPClientCapabilityProfile,
recordScheduledMCPToolAuthFailure,
} = require('@librechat/api');
const { CacheKeys, Constants } = require('librechat-data-provider');
const { getMCPManager, getMCPServersRegistry, getFlowStateManager } = require('~/config');
@ -136,6 +137,8 @@ async function loadMCPServerCatalogs({
* @param {import('@librechat/api').RequestScopedMCPConnectionStore} [params.requestScopedConnections]
* @param {Record<string, Record<string, string>>} [params.userMCPAuthMap]
* @param {import('@librechat/api').MCPServerCatalogRecoveryPolicy} [params.recoveryPolicy]
* @param {string | null} [params.streamId] - Owning generation's stream ID, if resumable.
* @param {number} [params.jobCreatedAt] - Owning generation epoch.
*/
async function reinitMCPServer({
user,
@ -157,6 +160,8 @@ async function reinitMCPServer({
oauthEnd,
recoveryPolicy,
mcpApps,
streamId,
jobCreatedAt,
}) {
const capabilityProfile = resolveMCPClientCapabilityProfile(mcpApps);
/** @type {MCPConnection | null} */
@ -466,6 +471,10 @@ async function reinitMCPServer({
return result;
} catch (error) {
if (isMCPInitializationError(error, signal)) {
await recordScheduledMCPToolAuthFailure(
{ error, streamId, jobCreatedAt, userId: user?.id, serverName },
() => require('~/server/services/Schedules').recordMCPToolAuthFailure,
);
throw error;
}
logger.error('[MCP Reinitialize] Error loading MCP tools; servers may still be initializing');

View file

@ -59,6 +59,9 @@ jest.mock('~/server/services/GraphTokenService', () => ({
jest.mock('~/cache', () => ({
getLogStores: jest.fn(() => ({})),
}));
jest.mock('~/server/services/Schedules', () => ({
recordMCPToolAuthFailure: jest.fn(async () => true),
}));
const { reinitMCPServer, loadMCPServerCatalogs } = require('./mcp');
@ -538,6 +541,75 @@ describe('reinitMCPServer — recovery of a server that failed inspection', () =
});
});
describe('scheduled MCP connection initialization', () => {
beforeEach(() => jest.clearAllMocks());
it('records a typed missing OBO provider before any tool instance exists', async () => {
const { OboTokenResolutionError } = require('@librechat/api');
const failure = new OboTokenResolutionError('missing_upstream_provider', 'Provider missing');
const receipt = require('~/server/services/Schedules').recordMCPToolAuthFailure;
mockGetConnection.mockRejectedValueOnce(failure);
await expect(
reinitMCPServer({
user: { id: 'owner' },
serverName: 'Graph',
serverConfig: {
type: 'streamable-http',
url: 'https://mcp.example.com',
source: 'yaml',
obo: { scopes: 'api://graph/.default' },
},
streamId: 'scheduled-conversation',
jobCreatedAt: 42,
}),
).rejects.toBe(failure);
expect(receipt).toHaveBeenCalledTimes(1);
expect(receipt).toHaveBeenCalledWith({
error: failure,
streamId: 'scheduled-conversation',
jobCreatedAt: 42,
userId: 'owner',
serverName: 'Graph',
});
});
it('preserves the connection error when the receipt store fails', async () => {
const { OboTokenResolutionError } = require('@librechat/api');
const failure = new OboTokenResolutionError('missing_upstream_provider', 'Provider missing');
const receipt = require('~/server/services/Schedules').recordMCPToolAuthFailure;
receipt.mockRejectedValueOnce(new Error('Mongo unavailable'));
mockGetConnection.mockRejectedValueOnce(failure);
await expect(
reinitMCPServer({
user: { id: 'owner' },
serverName: 'Graph',
serverConfig: { type: 'streamable-http', url: 'https://mcp.example.com', source: 'yaml' },
streamId: 'scheduled-conversation',
jobCreatedAt: 42,
}),
).rejects.toBe(failure);
});
it('does not construct the schedule facade for unrelated typed OBO failures', async () => {
const { OboTokenResolutionError } = require('@librechat/api');
const failure = new OboTokenResolutionError('session_refresh_failed', 'Retry later', true);
const receipt = require('~/server/services/Schedules').recordMCPToolAuthFailure;
mockGetConnection.mockRejectedValueOnce(failure);
await expect(
reinitMCPServer({
user: { id: 'owner' },
serverName: 'Graph',
serverConfig: { type: 'streamable-http', url: 'https://mcp.example.com', source: 'yaml' },
streamId: 'scheduled-conversation',
jobCreatedAt: 42,
}),
).rejects.toBe(failure);
expect(receipt).not.toHaveBeenCalled();
});
});
describe('reinitMCPServer — direct bearer authentication outcomes', () => {
it('preserves a typed rejection instead of reducing it to a generic result', async () => {
const { MCPAuthenticationRejectedError } = require('@librechat/api');

View file

@ -1838,6 +1838,9 @@ describe('ToolService - Action Capability Gating', () => {
expect(emittedStreamId).toBe(streamId);
expect(options).toEqual({ expectedCreatedAt: jobCreatedAt });
}
for (const [reinitInput] of reinitMCPServer.mock.calls) {
expect(reinitInput).toEqual(expect.objectContaining({ streamId, jobCreatedAt }));
}
});
it('should not expose cached MCP tool definitions when the registry lookup fails', async () => {
@ -2381,6 +2384,8 @@ describe('ToolService - Action Capability Gating', () => {
definitionsOnly: true,
signal,
upstreamTokenProvider: scheduledProvider,
streamId: 'scheduled-stream',
jobCreatedAt: 42,
});
expect(reinitMCPServer).toHaveBeenCalledWith(
@ -2388,6 +2393,8 @@ describe('ToolService - Action Capability Gating', () => {
serverName,
forceNew: true,
upstreamTokenProvider: scheduledProvider,
streamId: 'scheduled-stream',
jobCreatedAt: 42,
}),
);
});