🔕 fix: Log Lost Generation Predecessor Fences as Warnings (#16689)

A conditional generation create that loses its predecessor fence is an expected admission outcome: the store rejects before replacing anything, an automatic continuation defers without consuming an attempt, and a client restores its queued turn. Sibling background-tool wake-ups for one conversation race this fence by design, which logged every lost race as an initialization error with a stack.

Route the controller's initialization-failure log through a typed packages/api helper that logs a predecessor mismatch as a concise warning with the stream, conversation, continuation flag, and whether the current generation is active, and keeps every other failure at error level as redacted text.
This commit is contained in:
Danny Avila 2026-10-02 20:12:22 -04:00 • committed by GitHub
parent c5a5fbdaf1
commit 2529b22a66
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 369 additions and 4 deletions

View file

@ -82,7 +82,7 @@ jest.mock('@librechat/api', () => ({
resolvePersistableCodeEnvironmentDecision: (...args) =>
jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),
getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,
getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText,
logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,
startAgentProjectContextResolution:
jest.requireActual('@librechat/api').startAgentProjectContextResolution,
assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions,

View file

@ -308,7 +308,7 @@ jest.mock('@librechat/api', () => ({
resolvePersistableCodeEnvironmentDecision: (...args) =>
jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),
getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,
getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText,
logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,
resolveFailedTurnContent: jest.requireActual('@librechat/api').resolveFailedTurnContent,
getFailedTurnTraceFields: (...args) => mockGetFailedTurnTraceFields(...args),
startAgentProjectContextResolution:
@ -1385,6 +1385,65 @@ describe('ResumableAgentController resume metadata', () => {
);
});
it('defers a continuation that loses the create fence to a sibling and logs a warning', async () => {
mockGetMessages.mockResolvedValue([{ _id: 'persisted-parent' }]);
const { JobPredecessorMismatchError } = jest.requireActual('@librechat/api');
mockGenerationJobManager.createJob.mockRejectedValue(
new JobPredecessorMismatchError({
createdAt: 2000,
active: true,
verified: true,
status: 'running',
conversationId: 'conversation-123',
}),
);
const req = {
_isAgentTrigger: true,
user: { id: 'user-123' },
body: {
text: 'A background tool task is waiting to complete.',
messageId: 'wakeup-user-message',
parentMessageId: 'persisted-response_',
conversationId: 'conversation-123',
clientRequestId: 'trigger_sibling_wakeup',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
const res = createResumableResponse();
const initializeClient = jest.fn();
await AgentController(req, res, jest.fn(), initializeClient, null);
expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
'conversation-123',
'user-123',
'conversation-123',
expect.objectContaining({ rejectActivePredecessor: true }),
);
expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
expect(res.status).toHaveBeenCalledWith(409);
expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'PARENT_NOT_READY' }));
expect(mockLogger.warn).toHaveBeenCalledWith(
'[ResumableAgentController] Generation predecessor changed before creation',
{
streamId: 'conversation-123',
conversationId: 'conversation-123',
continuation: true,
active: true,
currentCreatedAt: 2000,
currentStatus: 'running',
},
);
expect(
mockLogger.error.mock.calls.some((call) =>
String(call[0]).startsWith('[ResumableAgentController] Initialization error:'),
),
).toBe(false);
expect(initializeClient).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
});
it('keeps request-scoped MCP connections until resumable initialization finishes', async () => {
const conversationId = 'conversation-123';
const disconnect = jest.fn().mockResolvedValue(undefined);
@ -3055,6 +3114,23 @@ describe('ResumableAgentController resume metadata', () => {
expect.objectContaining(DEFAULT_OWNED_CLAIM),
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
expect(mockLogger.warn).toHaveBeenCalledWith(
'[ResumableAgentController] Generation predecessor changed before creation',
{
streamId: 'conversation-123',
conversationId: 'conversation-123',
continuation: false,
active: true,
expectedPredecessorCreatedAt: 1000,
currentCreatedAt: 2000,
currentStatus: 'running',
},
);
expect(
mockLogger.error.mock.calls.some((call) =>
String(call[0]).startsWith('[ResumableAgentController] Initialization error:'),
),
).toBe(false);
});
it('returns a finite fail-closed mismatch when predecessor evidence expired', async () => {

View file

@ -39,7 +39,7 @@ const {
getAttachmentTitleText,
createMCPRuntimeRequestBody,
resolveRunCodeWorkspaces,
getSafeErrorText,
logGenerationStartFailure,
isAgentEventRetentionActive,
createAgentEventActorTurn,
createAgentEventActorDetachedActionLifecycle,
@ -3483,7 +3483,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
);
});
} catch (error) {
logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`);
logGenerationStartFailure(error, {
streamId,
conversationId,
continuation: isTriggerContinuation,
expectedPredecessorCreatedAt,
});
const initializationFailure = getInitializationFailure(error);
const streamStarted = res.headersSent;
try {

View file

@ -0,0 +1,189 @@
import { logger } from '@librechat/data-schemas';
import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
import { logGenerationStartFailure } from '../admission';
const STARTED = { generationProtocolVersion: 2 as const };
function createContinuationJob(store: InMemoryJobStore, streamId: string) {
return store.createJob(
streamId,
'owner-1',
streamId,
undefined,
STARTED,
undefined,
undefined,
undefined,
undefined,
undefined,
'continuation-create-attempt',
undefined,
true,
);
}
function createConditionalJob(
store: InMemoryJobStore,
streamId: string,
expectedPredecessorCreatedAt: number,
) {
return store.createJob(
streamId,
'owner-1',
streamId,
undefined,
STARTED,
undefined,
undefined,
undefined,
undefined,
undefined,
'conditional-create-attempt',
expectedPredecessorCreatedAt,
);
}
async function rejectionOf(promise: Promise<unknown>): Promise<unknown> {
try {
await promise;
} catch (error) {
return error;
}
throw new Error('Expected the create to be rejected');
}
describe('logGenerationStartFailure', () => {
let store: InMemoryJobStore;
let warn: jest.SpyInstance;
let error: jest.SpyInstance;
beforeEach(() => {
store = new InMemoryJobStore();
warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
error = jest.spyOn(logger, 'error').mockImplementation(() => logger);
});
afterEach(async () => {
jest.restoreAllMocks();
await store.destroy();
});
it('warns when a sibling continuation already won the conversation', async () => {
const streamId = 'sibling-wakeup-race';
const winner = await createContinuationJob(store, streamId);
const mismatch = await rejectionOf(createContinuationJob(store, streamId));
logGenerationStartFailure(mismatch, {
streamId,
conversationId: streamId,
continuation: true,
});
expect(error).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
'[ResumableAgentController] Generation predecessor changed before creation',
{
streamId,
conversationId: streamId,
continuation: true,
active: true,
currentCreatedAt: winner.createdAt,
currentStatus: 'running',
},
);
});
it('warns with the stale epoch when a queued send lost its fence to a settled turn', async () => {
const streamId = 'stale-queued-send';
const settled = await store.createJob(streamId, 'owner-1', streamId, undefined, STARTED);
await store.transitionStatus(streamId, {
from: 'running',
to: 'complete',
expectCreatedAt: settled.createdAt,
});
const expectedPredecessorCreatedAt = settled.createdAt - 1;
const mismatch = await rejectionOf(
createConditionalJob(store, streamId, expectedPredecessorCreatedAt),
);
logGenerationStartFailure(mismatch, {
streamId,
conversationId: streamId,
continuation: false,
expectedPredecessorCreatedAt,
});
expect(error).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalledWith(
'[ResumableAgentController] Generation predecessor changed before creation',
{
streamId,
conversationId: streamId,
continuation: false,
active: false,
expectedPredecessorCreatedAt,
currentCreatedAt: settled.createdAt,
currentStatus: 'complete',
},
);
});
it('treats a code-only mismatch the same way the 409 response does', () => {
const mismatch = Object.assign(new Error('predecessor changed'), {
code: 'GENERATION_PREDECESSOR_MISMATCH',
currentJob: { createdAt: 2000, status: 'requires_action', verified: false },
});
logGenerationStartFailure(mismatch, {
streamId: 'code-only',
conversationId: 'code-only',
continuation: false,
});
expect(error).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalledWith(
'[ResumableAgentController] Generation predecessor changed before creation',
{
streamId: 'code-only',
conversationId: 'code-only',
continuation: false,
active: true,
currentCreatedAt: 2000,
currentStatus: 'requires_action',
verified: false,
},
);
});
it('keeps genuine initialization failures at error level as redacted text', () => {
const failure = Object.assign(new Error('Agent not found'), { code: 'AGENT_NOT_FOUND' });
logGenerationStartFailure(failure, {
streamId: 'genuine-failure',
conversationId: 'genuine-failure',
continuation: true,
});
expect(warn).not.toHaveBeenCalled();
expect(error).toHaveBeenCalledTimes(1);
const [message, ...metadata] = error.mock.calls[0];
expect(metadata).toEqual([]);
expect(message).toMatch(
/^\[ResumableAgentController\] Initialization error: Error: Agent not found/,
);
});
it('logs a non-error rejection at error level', () => {
logGenerationStartFailure(undefined, {
streamId: 'unknown-failure',
conversationId: 'unknown-failure',
continuation: false,
});
expect(warn).not.toHaveBeenCalled();
expect(error).toHaveBeenCalledWith(
'[ResumableAgentController] Initialization error: UnknownError',
);
});
});

View file

@ -0,0 +1,92 @@
import { logger } from '@librechat/data-schemas';
import type { GenerationPredecessorState, JobStatus } from './interfaces/IJobStore';
import { JobPredecessorMismatchError } from './interfaces/IJobStore';
import { getSafeErrorText } from '../utils/errors';
const PREDECESSOR_MISMATCH_CODE: JobPredecessorMismatchError['code'] =
'GENERATION_PREDECESSOR_MISMATCH';
export interface GenerationStartFailureContext {
streamId: string;
conversationId: string;
/** Automatic continuation (trigger delivery, wake-up, queued turn); an active
* winner answers it with a deferrable `PARENT_NOT_READY`. */
continuation: boolean;
expectedPredecessorCreatedAt?: number;
}
type PredecessorMismatchState = Partial<GenerationPredecessorState>;
function isJobStatus(value: unknown): value is JobStatus {
return (
value === 'running' ||
value === 'requires_action' ||
value === 'complete' ||
value === 'error' ||
value === 'aborted'
);
}
/** Matches by `code` as well as class, mirroring how the controller selects its
* 409 response, so the log level and the response never disagree. */
function getPredecessorMismatchState(error: unknown): PredecessorMismatchState | undefined {
if (error instanceof JobPredecessorMismatchError) {
return error.currentJob;
}
if (error == null || typeof error !== 'object' || !('code' in error)) {
return undefined;
}
if (error.code !== PREDECESSOR_MISMATCH_CODE) {
return undefined;
}
const currentJob = 'currentJob' in error ? error.currentJob : undefined;
if (currentJob == null || typeof currentJob !== 'object') {
return {};
}
const createdAt = 'createdAt' in currentJob ? currentJob.createdAt : undefined;
const active = 'active' in currentJob ? currentJob.active : undefined;
const verified = 'verified' in currentJob ? currentJob.verified : undefined;
const status = 'status' in currentJob ? currentJob.status : undefined;
return {
...(typeof createdAt === 'number' && { createdAt }),
...(typeof active === 'boolean' && { active }),
...(typeof verified === 'boolean' && { verified }),
...(isJobStatus(status) && { status }),
};
}
/**
* Logs why a resumable generation failed to start.
*
* A predecessor-fence rejection is an expected admission outcome, not a fault:
* the store refused before replacing anything, an automatic continuation defers
* without consuming an attempt, and a client restores its queued turn. Sibling
* background-tool wake-ups for one conversation race this fence by design, so it
* logs a concise warning. Every other initialization failure stays an error.
*/
export function logGenerationStartFailure(
error: unknown,
context: GenerationStartFailureContext,
): void {
const currentJob = getPredecessorMismatchState(error);
if (currentJob === undefined) {
logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`);
return;
}
const active =
typeof currentJob.active === 'boolean'
? currentJob.active
: currentJob.status === 'running' || currentJob.status === 'requires_action';
logger.warn('[ResumableAgentController] Generation predecessor changed before creation', {
streamId: context.streamId,
conversationId: context.conversationId,
continuation: context.continuation,
active,
...(context.expectedPredecessorCreatedAt != null && {
expectedPredecessorCreatedAt: context.expectedPredecessorCreatedAt,
}),
...(currentJob.createdAt != null && { currentCreatedAt: currentJob.createdAt }),
...(currentJob.status != null && { currentStatus: currentJob.status }),
...(currentJob.verified === false && { verified: false }),
});
}

View file

@ -63,6 +63,9 @@ export {
} from './SteerRecovery';
export type { RecoveredSteerPayload } from './SteerRecovery';
export { logGenerationStartFailure } from './admission';
export type { GenerationStartFailureContext } from './admission';
export { createStreamServices } from './createStreamServices';
export type { StreamServicesConfig, StreamServices } from './createStreamServices';
export { filterPersistableAbortContent, hasPersistableAbortContent } from './abortContent';