diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js index 18a8828497..8955e99e7d 100644 --- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js +++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js @@ -82,7 +82,7 @@ jest.mock('@librechat/api', () => ({ resolvePersistableCodeEnvironmentDecision: (...args) => jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args), getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata, - getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText, + logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure, startAgentProjectContextResolution: jest.requireActual('@librechat/api').startAgentProjectContextResolution, assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions, diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index c8dcd02f81..5211dd610c 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -308,7 +308,7 @@ jest.mock('@librechat/api', () => ({ resolvePersistableCodeEnvironmentDecision: (...args) => jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args), getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata, - getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText, + logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure, resolveFailedTurnContent: jest.requireActual('@librechat/api').resolveFailedTurnContent, getFailedTurnTraceFields: (...args) => mockGetFailedTurnTraceFields(...args), startAgentProjectContextResolution: @@ -1385,6 +1385,65 @@ describe('ResumableAgentController resume metadata', () => { ); }); + it('defers a continuation that loses the create fence to a sibling and logs a warning', async () => { + mockGetMessages.mockResolvedValue([{ _id: 'persisted-parent' }]); + const { JobPredecessorMismatchError } = jest.requireActual('@librechat/api'); + mockGenerationJobManager.createJob.mockRejectedValue( + new JobPredecessorMismatchError({ + createdAt: 2000, + active: true, + verified: true, + status: 'running', + conversationId: 'conversation-123', + }), + ); + const req = { + _isAgentTrigger: true, + user: { id: 'user-123' }, + body: { + text: 'A background tool task is waiting to complete.', + messageId: 'wakeup-user-message', + parentMessageId: 'persisted-response_', + conversationId: 'conversation-123', + clientRequestId: 'trigger_sibling_wakeup', + endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } }, + }, + config: {}, + }; + const res = createResumableResponse(); + const initializeClient = jest.fn(); + + await AgentController(req, res, jest.fn(), initializeClient, null); + + expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith( + 'conversation-123', + 'user-123', + 'conversation-123', + expect.objectContaining({ rejectActivePredecessor: true }), + ); + expect(res.set).toHaveBeenCalledWith('Retry-After', '1'); + expect(res.status).toHaveBeenCalledWith(409); + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'PARENT_NOT_READY' })); + expect(mockLogger.warn).toHaveBeenCalledWith( + '[ResumableAgentController] Generation predecessor changed before creation', + { + streamId: 'conversation-123', + conversationId: 'conversation-123', + continuation: true, + active: true, + currentCreatedAt: 2000, + currentStatus: 'running', + }, + ); + expect( + mockLogger.error.mock.calls.some((call) => + String(call[0]).startsWith('[ResumableAgentController] Initialization error:'), + ), + ).toBe(false); + expect(initializeClient).not.toHaveBeenCalled(); + expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled(); + }); + it('keeps request-scoped MCP connections until resumable initialization finishes', async () => { const conversationId = 'conversation-123'; const disconnect = jest.fn().mockResolvedValue(undefined); @@ -3055,6 +3114,23 @@ describe('ResumableAgentController resume metadata', () => { expect.objectContaining(DEFAULT_OWNED_CLAIM), ); expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123'); + expect(mockLogger.warn).toHaveBeenCalledWith( + '[ResumableAgentController] Generation predecessor changed before creation', + { + streamId: 'conversation-123', + conversationId: 'conversation-123', + continuation: false, + active: true, + expectedPredecessorCreatedAt: 1000, + currentCreatedAt: 2000, + currentStatus: 'running', + }, + ); + expect( + mockLogger.error.mock.calls.some((call) => + String(call[0]).startsWith('[ResumableAgentController] Initialization error:'), + ), + ).toBe(false); }); it('returns a finite fail-closed mismatch when predecessor evidence expired', async () => { diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index f378f1ad83..9801525b40 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -39,7 +39,7 @@ const { getAttachmentTitleText, createMCPRuntimeRequestBody, resolveRunCodeWorkspaces, - getSafeErrorText, + logGenerationStartFailure, isAgentEventRetentionActive, createAgentEventActorTurn, createAgentEventActorDetachedActionLifecycle, @@ -3483,7 +3483,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit ); }); } catch (error) { - logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`); + logGenerationStartFailure(error, { + streamId, + conversationId, + continuation: isTriggerContinuation, + expectedPredecessorCreatedAt, + }); const initializationFailure = getInitializationFailure(error); const streamStarted = res.headersSent; try { diff --git a/packages/api/src/stream/__tests__/admission.spec.ts b/packages/api/src/stream/__tests__/admission.spec.ts new file mode 100644 index 0000000000..a39475f43c --- /dev/null +++ b/packages/api/src/stream/__tests__/admission.spec.ts @@ -0,0 +1,189 @@ +import { logger } from '@librechat/data-schemas'; +import { InMemoryJobStore } from '../implementations/InMemoryJobStore'; +import { logGenerationStartFailure } from '../admission'; + +const STARTED = { generationProtocolVersion: 2 as const }; + +function createContinuationJob(store: InMemoryJobStore, streamId: string) { + return store.createJob( + streamId, + 'owner-1', + streamId, + undefined, + STARTED, + undefined, + undefined, + undefined, + undefined, + undefined, + 'continuation-create-attempt', + undefined, + true, + ); +} + +function createConditionalJob( + store: InMemoryJobStore, + streamId: string, + expectedPredecessorCreatedAt: number, +) { + return store.createJob( + streamId, + 'owner-1', + streamId, + undefined, + STARTED, + undefined, + undefined, + undefined, + undefined, + undefined, + 'conditional-create-attempt', + expectedPredecessorCreatedAt, + ); +} + +async function rejectionOf(promise: Promise): Promise { + try { + await promise; + } catch (error) { + return error; + } + throw new Error('Expected the create to be rejected'); +} + +describe('logGenerationStartFailure', () => { + let store: InMemoryJobStore; + let warn: jest.SpyInstance; + let error: jest.SpyInstance; + + beforeEach(() => { + store = new InMemoryJobStore(); + warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger); + error = jest.spyOn(logger, 'error').mockImplementation(() => logger); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + await store.destroy(); + }); + + it('warns when a sibling continuation already won the conversation', async () => { + const streamId = 'sibling-wakeup-race'; + const winner = await createContinuationJob(store, streamId); + const mismatch = await rejectionOf(createContinuationJob(store, streamId)); + + logGenerationStartFailure(mismatch, { + streamId, + conversationId: streamId, + continuation: true, + }); + + expect(error).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + '[ResumableAgentController] Generation predecessor changed before creation', + { + streamId, + conversationId: streamId, + continuation: true, + active: true, + currentCreatedAt: winner.createdAt, + currentStatus: 'running', + }, + ); + }); + + it('warns with the stale epoch when a queued send lost its fence to a settled turn', async () => { + const streamId = 'stale-queued-send'; + const settled = await store.createJob(streamId, 'owner-1', streamId, undefined, STARTED); + await store.transitionStatus(streamId, { + from: 'running', + to: 'complete', + expectCreatedAt: settled.createdAt, + }); + const expectedPredecessorCreatedAt = settled.createdAt - 1; + const mismatch = await rejectionOf( + createConditionalJob(store, streamId, expectedPredecessorCreatedAt), + ); + + logGenerationStartFailure(mismatch, { + streamId, + conversationId: streamId, + continuation: false, + expectedPredecessorCreatedAt, + }); + + expect(error).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + '[ResumableAgentController] Generation predecessor changed before creation', + { + streamId, + conversationId: streamId, + continuation: false, + active: false, + expectedPredecessorCreatedAt, + currentCreatedAt: settled.createdAt, + currentStatus: 'complete', + }, + ); + }); + + it('treats a code-only mismatch the same way the 409 response does', () => { + const mismatch = Object.assign(new Error('predecessor changed'), { + code: 'GENERATION_PREDECESSOR_MISMATCH', + currentJob: { createdAt: 2000, status: 'requires_action', verified: false }, + }); + + logGenerationStartFailure(mismatch, { + streamId: 'code-only', + conversationId: 'code-only', + continuation: false, + }); + + expect(error).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + '[ResumableAgentController] Generation predecessor changed before creation', + { + streamId: 'code-only', + conversationId: 'code-only', + continuation: false, + active: true, + currentCreatedAt: 2000, + currentStatus: 'requires_action', + verified: false, + }, + ); + }); + + it('keeps genuine initialization failures at error level as redacted text', () => { + const failure = Object.assign(new Error('Agent not found'), { code: 'AGENT_NOT_FOUND' }); + + logGenerationStartFailure(failure, { + streamId: 'genuine-failure', + conversationId: 'genuine-failure', + continuation: true, + }); + + expect(warn).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledTimes(1); + const [message, ...metadata] = error.mock.calls[0]; + expect(metadata).toEqual([]); + expect(message).toMatch( + /^\[ResumableAgentController\] Initialization error: Error: Agent not found/, + ); + }); + + it('logs a non-error rejection at error level', () => { + logGenerationStartFailure(undefined, { + streamId: 'unknown-failure', + conversationId: 'unknown-failure', + continuation: false, + }); + + expect(warn).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith( + '[ResumableAgentController] Initialization error: UnknownError', + ); + }); +}); diff --git a/packages/api/src/stream/admission.ts b/packages/api/src/stream/admission.ts new file mode 100644 index 0000000000..18dcd81414 --- /dev/null +++ b/packages/api/src/stream/admission.ts @@ -0,0 +1,92 @@ +import { logger } from '@librechat/data-schemas'; +import type { GenerationPredecessorState, JobStatus } from './interfaces/IJobStore'; +import { JobPredecessorMismatchError } from './interfaces/IJobStore'; +import { getSafeErrorText } from '../utils/errors'; + +const PREDECESSOR_MISMATCH_CODE: JobPredecessorMismatchError['code'] = + 'GENERATION_PREDECESSOR_MISMATCH'; + +export interface GenerationStartFailureContext { + streamId: string; + conversationId: string; + /** Automatic continuation (trigger delivery, wake-up, queued turn); an active + * winner answers it with a deferrable `PARENT_NOT_READY`. */ + continuation: boolean; + expectedPredecessorCreatedAt?: number; +} + +type PredecessorMismatchState = Partial; + +function isJobStatus(value: unknown): value is JobStatus { + return ( + value === 'running' || + value === 'requires_action' || + value === 'complete' || + value === 'error' || + value === 'aborted' + ); +} + +/** Matches by `code` as well as class, mirroring how the controller selects its + * 409 response, so the log level and the response never disagree. */ +function getPredecessorMismatchState(error: unknown): PredecessorMismatchState | undefined { + if (error instanceof JobPredecessorMismatchError) { + return error.currentJob; + } + if (error == null || typeof error !== 'object' || !('code' in error)) { + return undefined; + } + if (error.code !== PREDECESSOR_MISMATCH_CODE) { + return undefined; + } + const currentJob = 'currentJob' in error ? error.currentJob : undefined; + if (currentJob == null || typeof currentJob !== 'object') { + return {}; + } + const createdAt = 'createdAt' in currentJob ? currentJob.createdAt : undefined; + const active = 'active' in currentJob ? currentJob.active : undefined; + const verified = 'verified' in currentJob ? currentJob.verified : undefined; + const status = 'status' in currentJob ? currentJob.status : undefined; + return { + ...(typeof createdAt === 'number' && { createdAt }), + ...(typeof active === 'boolean' && { active }), + ...(typeof verified === 'boolean' && { verified }), + ...(isJobStatus(status) && { status }), + }; +} + +/** + * Logs why a resumable generation failed to start. + * + * A predecessor-fence rejection is an expected admission outcome, not a fault: + * the store refused before replacing anything, an automatic continuation defers + * without consuming an attempt, and a client restores its queued turn. Sibling + * background-tool wake-ups for one conversation race this fence by design, so it + * logs a concise warning. Every other initialization failure stays an error. + */ +export function logGenerationStartFailure( + error: unknown, + context: GenerationStartFailureContext, +): void { + const currentJob = getPredecessorMismatchState(error); + if (currentJob === undefined) { + logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`); + return; + } + const active = + typeof currentJob.active === 'boolean' + ? currentJob.active + : currentJob.status === 'running' || currentJob.status === 'requires_action'; + logger.warn('[ResumableAgentController] Generation predecessor changed before creation', { + streamId: context.streamId, + conversationId: context.conversationId, + continuation: context.continuation, + active, + ...(context.expectedPredecessorCreatedAt != null && { + expectedPredecessorCreatedAt: context.expectedPredecessorCreatedAt, + }), + ...(currentJob.createdAt != null && { currentCreatedAt: currentJob.createdAt }), + ...(currentJob.status != null && { currentStatus: currentJob.status }), + ...(currentJob.verified === false && { verified: false }), + }); +} diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts index 504fafa586..b709ad21ef 100644 --- a/packages/api/src/stream/index.ts +++ b/packages/api/src/stream/index.ts @@ -63,6 +63,9 @@ export { } from './SteerRecovery'; export type { RecoveredSteerPayload } from './SteerRecovery'; +export { logGenerationStartFailure } from './admission'; +export type { GenerationStartFailureContext } from './admission'; + export { createStreamServices } from './createStreamServices'; export type { StreamServicesConfig, StreamServices } from './createStreamServices'; export { filterPersistableAbortContent, hasPersistableAbortContent } from './abortContent';