🚦 feat: Dispatch and Recover Background Responses (#17600)

* 🚦 feat: Dispatch and Recover Background Responses

* fix: guard response publication and retirement by durable identity

* fix: normalize legacy response tenant identity at execution boundaries

* style: order response scope imports

* style: Format Restacked Metadata Fields

* test: Enroll Ordinary Abort Without External Ownership

* test: Omit Ordinary Terminal Ownership Option

* fix: Keep Terminal Reads and Worker Retries Under Their Owners
This commit is contained in:
Dustin Healy 2026-11-10 21:03:59 -08:00 • committed by GitHub
parent 7e51963f2b
commit d3fc693cce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 4872 additions and 199 deletions

View file

@ -0,0 +1,113 @@
import { tenantStorage } from '@librechat/data-schemas';
import { PermissionBits, Permissions, PermissionTypes } from 'librechat-data-provider';
import type { BackgroundResponseAuthorityDependencies } from './authority';
import { createBackgroundResponseAuthority } from './authority';
const owner = { userId: 'background-owner', tenantId: 'background-tenant' };
function fixture(options: { tenantId?: string | null } = { tenantId: owner.tenantId }) {
const user = {
_id: owner.userId,
role: 'USER',
tenantId: options.tenantId,
provider: 'local',
twoFactorEnabled: false,
};
const deps: BackgroundResponseAuthorityDependencies = {
findUser: jest.fn().mockResolvedValue(user),
getKey: jest.fn().mockResolvedValue({}),
getRole: jest.fn().mockResolvedValue({
permissions: { [PermissionTypes.REMOTE_AGENTS]: { [Permissions.USE]: true } },
}),
getConfig: jest.fn().mockResolvedValue({
endpoints: {
agents: {
remoteApi: { background: { enabled: true }, auth: { apiKey: { enabled: true } } },
},
},
}),
getAgentPermissions: jest.fn().mockResolvedValue(PermissionBits.VIEW),
isPrincipalActive: jest.fn().mockResolvedValue(true),
};
return { deps, user, authorize: createBackgroundResponseAuthority(deps) };
}
const originalPolicy = process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION;
afterEach(() => {
if (originalPolicy === undefined) delete process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION;
else process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = originalPolicy;
});
it('rechecks shared enrollment policy when enforcement changes after queue admission', async () => {
const { user, deps, authorize } = fixture();
process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'false';
expect(await authorize(owner, 'agent', 'original-key')).toBe(true);
process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true';
jest.mocked(deps.getKey).mockClear();
expect(await authorize(owner, 'agent', 'original-key')).toBe(false);
expect(deps.getKey).not.toHaveBeenCalled();
user.twoFactorEnabled = true;
expect(await authorize(owner, 'agent', 'original-key')).toBe(true);
});
it('uses an already-loaded user while enforcing enrollment before background grants', async () => {
const { deps, authorize } = fixture();
process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true';
const user = await deps.findUser(owner.userId);
if (!user) throw new Error('Fixture missing user');
jest.mocked(deps.findUser).mockClear();
expect(
await tenantStorage.run(owner, () => authorize(owner, 'agent', 'original-key', user)),
).toBe(false);
expect(deps.findUser).not.toHaveBeenCalled();
expect(deps.getKey).not.toHaveBeenCalled();
});
it('accepts the host enrollment policy dependency without bypassing it for a loaded user', async () => {
const { deps } = fixture();
deps.enrollmentRequired = jest.fn().mockReturnValue(true);
const user = await deps.findUser(owner.userId);
if (!user) throw new Error('Fixture missing user');
expect(await createBackgroundResponseAuthority(deps)(owner, 'agent', 'original-key', user)).toBe(
false,
);
expect(deps.enrollmentRequired).toHaveBeenCalledWith(user);
expect(deps.getKey).not.toHaveBeenCalled();
});
it('preserves federated policy exceptions and tenant isolation', async () => {
const { user, authorize } = fixture();
process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true';
user.provider = 'openid';
expect(await authorize(owner, 'agent', 'original-key')).toBe(true);
expect(
await tenantStorage.run({ tenantId: 'other-tenant' }, () =>
authorize(owner, 'agent', 'original-key'),
),
).toBe(false);
});
it.each([undefined, null, ''])(
'accepts a legacy tenantless user field %p without changing named authority',
async (tenantId) => {
const { deps, authorize } = fixture({ tenantId });
deps.enrollmentRequired = () => false;
const principal = { userId: owner.userId };
expect(await tenantStorage.run(principal, () => authorize(principal, 'agent', 'key'))).toBe(
true,
);
const loaded = await deps.findUser(principal.userId);
if (!loaded) throw new Error('Fixture missing user');
jest.mocked(deps.findUser).mockClear();
expect(await authorize({ ...principal, tenantId: '' }, 'agent', 'key', loaded)).toBe(true);
expect(deps.findUser).not.toHaveBeenCalled();
expect(await authorize(owner, 'agent', 'key', loaded)).toBe(false);
},
);
it.each(['_', '__SYSTEM__'])(
'rejects internal owner partition %p before principal lookup',
async (tenantId) => {
const { deps, authorize } = fixture({ tenantId });
expect(await authorize({ ...owner, tenantId }, 'agent', 'key')).toBe(false);
expect(deps.findUser).not.toHaveBeenCalled();
},
);

View file

@ -0,0 +1,90 @@
import { PermissionBits, Permissions, PermissionTypes } from 'librechat-data-provider';
import { getTenantId, SYSTEM_TENANT_ID, tenantStorage } from '@librechat/data-schemas';
import type {
AppConfig,
IUser,
IRole,
ResponseOwner,
AgentApiKeyBinding,
} from '@librechat/data-schemas';
import { isTwoFactorEnrollmentRequired } from '~/auth/twoFactor';
import { responseTenantId, isResponseTenantId } from './scope';
export interface BackgroundResponseAuthorityDependencies {
enrollmentRequired?: typeof isTwoFactorEnrollmentRequired;
findUser: (userId: string) => Promise<IUser | null>;
getKey: (
keyId: string,
userId: string,
) => Promise<{ expiresAt?: Date; binding?: AgentApiKeyBinding } | null>;
authorizeBinding?: (
binding: AgentApiKeyBinding,
keyId: string,
owner: ResponseOwner,
agentId: string,
) => Promise<boolean>;
getRole: (role: string) => Promise<IRole | null>;
getConfig: (user: IUser) => Promise<AppConfig>;
getAgentPermissions: (agentId: string, user: { id: string; role: string }) => Promise<number>;
isPrincipalActive: (userId: string) => Promise<boolean>;
now?: () => number;
}
/** Background work retains its original credential's authority, not the worker's signing key. */
export function createBackgroundResponseAuthority(deps: BackgroundResponseAuthorityDependencies) {
return async (
owner: ResponseOwner,
agentId: string,
sourceKeyId?: string,
loadedUser?: IUser,
): Promise<boolean> => {
if (!sourceKeyId || !isResponseTenantId(owner.tenantId)) return false;
const tenantId = responseTenantId(owner.tenantId);
const currentTenant = getTenantId();
if (
currentTenant != null &&
currentTenant !== SYSTEM_TENANT_ID &&
responseTenantId(currentTenant) !== tenantId
)
return false;
return tenantStorage.run({ tenantId, userId: owner.userId }, async () => {
const user = loadedUser ?? (await deps.findUser(owner.userId));
if (
!user ||
String(user._id) !== owner.userId ||
!isResponseTenantId(user.tenantId) ||
responseTenantId(user.tenantId) !== tenantId ||
!user.role ||
(deps.enrollmentRequired ?? isTwoFactorEnrollmentRequired)(user)
)
return false;
const [key, role, config, active, permissions] = await Promise.all([
deps.getKey(sourceKeyId, owner.userId),
deps.getRole(user.role),
deps.getConfig(user),
deps.isPrincipalActive(owner.userId),
deps.getAgentPermissions(agentId, { id: owner.userId, role: user.role }),
]);
const remote = config.endpoints?.agents?.remoteApi;
if (
key?.binding &&
!(await deps.authorizeBinding?.(
key.binding,
sourceKeyId,
{ userId: owner.userId, tenantId },
agentId,
))
)
return false;
return Boolean(
key &&
(!key.expiresAt || key.expiresAt.getTime() > (deps.now ?? Date.now)()) &&
active &&
role?.permissions?.[PermissionTypes.REMOTE_AGENTS]?.[Permissions.USE] === true &&
remote?.background?.enabled === true &&
remote?.auth?.apiKey?.enabled !== false &&
(permissions & PermissionBits.VIEW) === PermissionBits.VIEW,
);
});
};
}

View file

@ -0,0 +1,509 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import { createResponseStorage, tenantStorage, runAsSystem } from '@librechat/data-schemas';
import type {
ResponseStorage,
StoredResponseRecord,
ResponseOwner,
ResponseTerminalReceipt,
} from '@librechat/data-schemas';
import type { BackgroundResponseDependencies } from './background';
import type { ResponsesRunEnvelope } from '../envelope';
import type { Response } from './types';
import {
buildAggregatedResponse,
createResponseAggregator,
createResponseContext,
} from './service';
import { createBackgroundResponseService } from './background';
import { createResponseCompletion } from './completion';
import { createAgentRunEnvelope } from '../envelope';
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
const owner = { userId: 'owner', tenantId: 'tenant-a' };
const scope = <T>(work: () => Promise<T>) => tenantStorage.run({ tenantId: owner.tenantId }, work);
const envelope = (payload: Partial<ResponsesRunEnvelope['payload']> = {}) =>
createAgentRunEnvelope({
protocol: 'responses',
requestId: 'request',
receivedAt: 100,
principal: { id: owner.userId, tenantId: owner.tenantId },
payload: { model: 'agent', input: 'Hello', background: true, store: true, ...payload },
});
function completed(record: StoredResponseRecord): Response {
const aggregator = createResponseAggregator();
aggregator.addText('# Persisted response\n\n**Full answer.**');
return {
...buildAggregatedResponse(
createResponseContext({ model: record.agentId, input: 'Hello' }, record.responseId),
aggregator,
),
background: true,
store: true,
};
}
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 1 },
instanceOpts: [{ args: ['--nounixsocket'] }],
});
await mongoose.connect(mongo.getUri(), { autoIndex: false });
storage = createResponseStorage(mongoose);
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.ResponseScope?.deleteMany({});
});
});
function fixture() {
const deliveries = new Map<string, StoredResponseRecord>();
const cleanupCheckpoint = jest.fn(async () => {});
const releaseExecution = jest.fn(async () => {});
const completion = createResponseCompletion({ storage, cleanupCheckpoint, releaseExecution });
const deps: BackgroundResponseDependencies = {
storage,
authorize: async (principal, agentId) =>
principal.userId === owner.userId && agentId === 'agent',
resolveContinuation: async () => ({ conversationId: 'conversation', parentMessageId: 'root' }),
enqueue: async (record) => {
deliveries.set(record.deliveryKey!, record);
},
observe: async () => null,
recoverTerminal: completion.finish,
cancel: async () => {},
now: () => 200,
};
return {
deps,
deliveries,
completion,
cleanupCheckpoint,
releaseExecution,
service: createBackgroundResponseService(deps),
};
}
async function admit(record: StoredResponseRecord): Promise<StoredResponseRecord> {
const result = await storage.publish(record, record.responseId, record.revision, {
status: 'in_progress',
generationCreatedAt: 10,
responseJSON: JSON.stringify({
...completed(record),
status: 'in_progress',
completed_at: null,
output: [],
}),
});
if (!result.ok) throw new Error('Fixture admission failed');
return result.record;
}
function terminalReceipt(
record: StoredResponseRecord,
status: ResponseTerminalReceipt['status'] = 'completed',
) {
if (record.generationCreatedAt == null) throw new Error('Fixture generation missing');
const response: Response = { ...completed(record), status };
const receipt: ResponseTerminalReceipt = {
generationCreatedAt: record.generationCreatedAt,
providerExecutionId: 'provider',
ownerId: 'worker',
status,
responseJSON: JSON.stringify(response),
checkpointNamespace: 'checkpoint',
};
return { response, receipt };
}
it('reserves a generic response and preserves all validated execution settings', () =>
scope(async () => {
const { service, deliveries } = fixture();
const request = envelope({
instructions: 'Keep instructions',
temperature: 0.4,
manualSkills: ['skill'],
max_output_tokens: 1234,
metadata: { purpose: 'test' },
});
const result = await service.create(request, 'stable-key');
expect(result).toMatchObject({
ok: true,
response: { status: 'queued', background: true, store: true, output: [] },
});
const record = [...deliveries.values()][0];
expect(JSON.parse(record.requestJSON)).toEqual(request.payload);
expect(record.agentId).toBe('agent');
}));
it('replays the same identity, including after an uncertain queue publication', () =>
scope(async () => {
const { service, deps, deliveries } = fixture();
const enqueue = deps.enqueue;
let first = true;
deps.enqueue = async (record) => {
await enqueue(record);
if (first) {
first = false;
throw new Error('queue response lost');
}
};
await expect(service.create(envelope(), 'same')).rejects.toThrow('queue response lost');
const result = await service.create(envelope(), 'same');
expect(result.ok).toBe(true);
expect(deliveries.size).toBe(1);
const record = [...deliveries.values()][0];
if (result.ok) expect(result.response.id).toBe(record.responseId);
}));
it('canonicalizes nested request key order but rejects changed request values', () =>
scope(async () => {
const { service, deliveries } = fixture();
const first = await service.create(envelope({ metadata: { a: '1', b: '2' } }), 'same');
const second = await service.create(envelope({ metadata: { b: '2', a: '1' } }), 'same');
expect(second).toEqual(first);
expect(await service.create(envelope({ metadata: { a: 'changed', b: '2' } }), 'same')).toEqual({
ok: false,
status: 409,
code: 'idempotency_conflict',
});
expect(deliveries.size).toBe(1);
}));
it('denies revoked authority before reserve, observation or cancellation', () =>
scope(async () => {
const { service, deps, deliveries } = fixture();
await service.create(envelope(), 'same');
const record = [...deliveries.values()][0];
deps.authorize = async () => false;
const observe = jest.spyOn(deps, 'observe');
const cancel = jest.spyOn(deps, 'cancel');
expect(await service.retrieve(owner, record.responseId)).toEqual({
ok: false,
status: 404,
code: 'response_not_found',
});
expect(await service.cancel(owner, record.responseId)).toEqual({
ok: false,
status: 404,
code: 'response_not_found',
});
expect(await service.create(envelope(), 'different')).toEqual({
ok: false,
status: 403,
code: 'permission_denied',
});
expect(observe).not.toHaveBeenCalled();
expect(cancel).not.toHaveBeenCalled();
expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1);
}));
it.each(['completed', 'failed', 'incomplete', 'cancelled'] as const)(
'defers an observed %s result until the terminal producer commits its journal',
(status) =>
scope(async () => {
const f = fixture();
await f.service.create(envelope(), 'same');
const record = await admit([...f.deliveries.values()][0]);
const { response, receipt } = terminalReceipt(record, status);
const observe = jest
.spyOn(f.deps, 'observe')
.mockResolvedValue({ response, generationCreatedAt: 10 });
const pending = await f.service.retrieve(owner, record.responseId);
expect(pending).toMatchObject({ ok: true, response: { status: 'in_progress', output: [] } });
expect((await storage.read(owner, record.responseId))?.status).toBe('in_progress');
expect(f.cleanupCheckpoint).not.toHaveBeenCalled();
expect(f.releaseExecution).not.toHaveBeenCalled();
await f.completion.stageAndFinish(record, receipt);
expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response });
expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response });
expect(observe).toHaveBeenCalledTimes(1);
expect(f.cleanupCheckpoint).toHaveBeenCalledTimes(1);
expect(f.releaseExecution).toHaveBeenCalled();
}),
);
it('persists cancellation intent before signalling and waits for the cancelled journal', () =>
scope(async () => {
const f = fixture();
await f.service.create(envelope(), 'same');
const record = await admit([...f.deliveries.values()][0]);
f.deps.cancel = async (current) => {
expect((await storage.read(owner, current.responseId))?.cancellationRequestedAt).toBe(200);
};
expect(await f.service.cancel(owner, record.responseId)).toMatchObject({
ok: true,
response: { status: 'in_progress' },
});
const { response, receipt } = terminalReceipt(record, 'cancelled');
f.deps.observe = async () => ({ response, generationCreatedAt: 10 });
expect(await f.service.retrieve(owner, record.responseId)).toMatchObject({
ok: true,
response: { status: 'in_progress' },
});
await f.completion.stageAndFinish(record, receipt);
expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response });
}));
it.each(['retrieve', 'replay', 'cancel'] as const)(
'routes %s of a pending receipt through completion without restarting or cancelling work',
(entry) =>
scope(async () => {
const f = fixture();
await f.service.create(envelope(), 'same');
const record = await admit([...f.deliveries.values()][0]);
const { response, receipt } = terminalReceipt(record);
const staged = await storage.stageTerminal(
owner,
record.responseId,
{ execution: record },
receipt,
);
if (!staged.ok) throw new Error('Fixture staging failed');
const enqueue = jest.spyOn(f.deps, 'enqueue');
const cancel = jest.spyOn(f.deps, 'cancel');
const observe = jest.spyOn(f.deps, 'observe');
const actions = {
retrieve: () => f.service.retrieve(owner, record.responseId),
replay: () => f.service.create(envelope(), 'same'),
cancel: () => f.service.cancel(owner, record.responseId),
};
const result = await actions[entry]();
expect(result).toEqual({ ok: true, response });
expect(enqueue).not.toHaveBeenCalled();
expect(cancel).not.toHaveBeenCalled();
expect(observe).not.toHaveBeenCalled();
expect(f.cleanupCheckpoint).toHaveBeenCalledTimes(1);
expect(f.releaseExecution).toHaveBeenCalledTimes(1);
expect(
(await storage.read(owner, record.responseId))?.cancellationRequestedAt,
).toBeUndefined();
}),
);
it('recovers a journal that wins a progress CAS race and waits for cleanup before returning its result', () =>
scope(async () => {
const f = fixture();
await f.service.create(envelope(), 'same');
const record = await admit([...f.deliveries.values()][0]);
const { response, receipt } = terminalReceipt(record);
f.deps.observe = async () => ({
response: { ...response, status: 'in_progress' },
generationCreatedAt: 10,
});
let releaseCleanup!: () => void;
let enteredCleanup!: () => void;
const gate = new Promise<void>((resolve) => {
releaseCleanup = resolve;
});
const entered = new Promise<void>((resolve) => {
enteredCleanup = resolve;
});
f.cleanupCheckpoint.mockImplementation(async () => {
enteredCleanup();
await gate;
});
const publish = storage.publish.bind(storage);
const spy = jest.spyOn(storage, 'publish').mockImplementationOnce(async (...args) => {
const staged = await storage.stageTerminal(
owner,
record.responseId,
{ execution: record },
receipt,
);
if (!staged.ok) throw new Error('Fixture staging race failed');
return publish(...args);
});
const read = f.service.retrieve(owner, record.responseId);
let settled = false;
void read.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
try {
await Promise.race([
entered,
read.then(() => {
throw new Error('Read bypassed cleanup');
}),
]);
expect(settled).toBe(false);
expect(f.releaseExecution).not.toHaveBeenCalled();
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(receipt);
releaseCleanup();
expect(await read).toEqual({ ok: true, response });
expect(f.releaseExecution).toHaveBeenCalledTimes(1);
} finally {
releaseCleanup();
await Promise.allSettled([read]);
spy.mockRestore();
}
}));
it('withholds terminal output through checkpoint and execution-release failures', () =>
scope(async () => {
const f = fixture();
await f.service.create(envelope(), 'same');
const record = await admit([...f.deliveries.values()][0]);
const { response, receipt } = terminalReceipt(record);
const staged = await storage.stageTerminal(
owner,
record.responseId,
{ execution: record },
receipt,
);
if (!staged.ok) throw new Error('Fixture staging failed');
f.cleanupCheckpoint.mockRejectedValueOnce(new Error('checkpoint unavailable'));
await expect(f.service.retrieve(owner, record.responseId)).rejects.toThrow(
'checkpoint unavailable',
);
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(receipt);
expect(f.releaseExecution).not.toHaveBeenCalled();
f.releaseExecution.mockRejectedValueOnce(new Error('release unavailable'));
await expect(f.service.retrieve(owner, record.responseId)).rejects.toThrow(
'release unavailable',
);
expect((await storage.read(owner, record.responseId))?.status).toBe('completed');
expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response });
expect(f.cleanupCheckpoint).toHaveBeenCalledTimes(2);
expect(f.releaseExecution).toHaveBeenCalledTimes(2);
}));
it('does not let a stale progress observation replace an enrolled generation or expose its output', () =>
scope(async () => {
const f = fixture();
await f.service.create(envelope(), 'same');
const record = await admit([...f.deliveries.values()][0]);
f.deps.observe = async () => ({
response: { ...completed(record), status: 'in_progress' },
generationCreatedAt: 20,
});
expect(await f.service.retrieve(owner, record.responseId)).toMatchObject({
ok: true,
response: { output: [] },
});
expect((await storage.read(owner, record.responseId))?.generationCreatedAt).toBe(10);
}));
it('rejects a runtime result carrying a different response identity', () =>
scope(async () => {
const { service, deps, deliveries } = fixture();
await service.create(envelope(), 'same');
const record = [...deliveries.values()][0];
deps.observe = async () => ({ response: { ...completed(record), id: 'resp_other' } });
await expect(service.retrieve(owner, record.responseId)).rejects.toThrow(
'execution identity changed',
);
expect((await storage.read(owner, record.responseId))?.status).toBe('queued');
}));
it.each([{ store: false }, { stream: true }, { background: false }])(
'rejects unsupported background combinations before publication: %p',
(payload) =>
scope(async () => {
const { service, deliveries } = fixture();
expect(await service.create(envelope(payload), 'same')).toEqual({
ok: false,
status: 400,
code: 'invalid_background_request',
});
expect(deliveries.size).toBe(0);
expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0);
}),
);
it.each(['queued', 'pending', 'completed'] as const)(
'reauthorizes the original source key before replaying a %s record with a replacement key',
(state) =>
scope(async () => {
const { deps, service, deliveries } = fixture();
const allowed = new Set(['key-a', 'key-b']);
const authorize = jest.fn(
async (_owner: ResponseOwner, _agent: string, key?: string) =>
key != null && allowed.has(key),
);
deps.authorize = authorize;
expect((await service.create(envelope(), 'same-request', 'key-a')).ok).toBe(true);
let record = [...deliveries.values()][0];
const response = completed(record);
if (state === 'completed') {
expect(
(
await storage.publish(owner, record.responseId, record.revision, {
status: 'completed',
responseJSON: JSON.stringify(response),
})
).ok,
).toBe(true);
} else if (state === 'pending') {
const running = await storage.publish(owner, record.responseId, record.revision, {
status: 'in_progress',
generationCreatedAt: 10,
responseJSON: JSON.stringify({ ...response, status: 'in_progress' }),
});
if (!running.ok) throw new Error('Publication failed');
record = running.record;
expect(
(
await storage.stageTerminal(owner, record.responseId, record.revision, {
generationCreatedAt: 10,
providerExecutionId: 'provider',
status: 'completed',
responseJSON: JSON.stringify(response),
})
).ok,
).toBe(true);
}
deliveries.clear();
deps.enqueue = jest.fn();
deps.observe = jest.fn();
deps.recoverTerminal = jest.fn();
allowed.delete('key-a');
authorize.mockClear();
expect(await service.create(envelope(), 'same-request', 'key-b')).toEqual({
ok: false,
status: 403,
code: 'permission_denied',
});
expect(authorize.mock.calls.map((call) => call[2])).toEqual(['key-b', 'key-a']);
expect(deps.enqueue).not.toHaveBeenCalled();
expect(deps.observe).not.toHaveBeenCalled();
expect(deps.recoverTerminal).not.toHaveBeenCalled();
expect(deliveries.size).toBe(0);
}),
);
it('does not add a second source-authority lookup for a newly reserved response', () =>
scope(async () => {
const { deps, service } = fixture();
deps.authorize = jest.fn().mockResolvedValue(true);
expect((await service.create(envelope(), 'new-request', 'key')).ok).toBe(true);
expect(deps.authorize).toHaveBeenCalledTimes(1);
}));
it('rejects JavaScript composition without a completion or execution-release owner before any reservation', async () => {
const f = fixture();
expect(() =>
Reflect.apply(createBackgroundResponseService, undefined, [
{ ...f.deps, recoverTerminal: undefined },
]),
).toThrow('completion owner');
expect(() =>
Reflect.apply(createResponseCompletion, undefined, [
{ storage, cleanupCheckpoint: async () => {} },
]),
).toThrow('cleanup and release owners');
expect(await runAsSystem(() => mongoose.models.StoredResponse.countDocuments({}))).toBe(0);
});

View file

@ -0,0 +1,213 @@
import { createHash, randomUUID } from 'node:crypto';
import type { ResponseOwner, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { ResponsesRunEnvelope, JsonValue } from '../envelope';
import type { ResponseCompletionRecord } from './completion';
import type { Response } from './types';
import {
createResponseContext,
createResponseAggregator,
buildAggregatedResponse,
} from './service';
export type BackgroundResponseResult =
| { ok: true; response: Response }
| { ok: false; status: 400 | 403 | 404 | 409; code: string };
/** The host supplies the existing durable queue and Responses execution runtime. */
export interface BackgroundResponseDependencies {
storage: ResponseStorage;
/** Recheck current credential/owner/Agent authority; possession of a response ID is not authority. */
authorize: (owner: ResponseOwner, agentId: string, sourceKeyId?: string) => Promise<boolean>;
resolveContinuation: (
owner: ResponseOwner,
previousResponseId?: string,
) => Promise<{
conversationId: string;
parentMessageId: string;
expiredAt?: Date;
isTemporary?: boolean;
}>;
/** Idempotent queue publication under record.deliveryKey; MUST NOT execute inline or mint a new identity on retry. */
enqueue: (record: StoredResponseRecord) => Promise<void>;
deliveryKey?: (record: StoredResponseRecord) => string;
/** Progress-only projection. A terminal observation is a hint to reread the journal;
* generationCreatedAt is a consistency witness, never enrollment authority. */
observe: (record: StoredResponseRecord) => Promise<{
response: Response;
generationCreatedAt?: number;
} | null>;
/** Completion journal owns cleanup and release before any terminal view becomes public. */
recoverTerminal: (record: StoredResponseRecord) => Promise<ResponseCompletionRecord>;
/** Signal exact execution; the completion journal confirms its final outcome. Never claim a missing job stopped. */
cancel: (record: StoredResponseRecord) => Promise<void>;
now?: () => number;
}
export function canonicalResponseJSON(request: object): string {
return JSON.stringify(request, (_key: string, value: JsonValue) => {
if (value === null || typeof value !== 'object' || Array.isArray(value)) return value;
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, value[key]]),
);
});
}
function terminal(status: StoredResponseRecord['status']) {
return ['completed', 'failed', 'incomplete', 'cancelled'].includes(status);
}
export function buildPendingBackgroundResponse(
record: StoredResponseRecord,
envelope?: ResponsesRunEnvelope,
): Response {
const request =
envelope?.payload ?? (JSON.parse(record.requestJSON) as ResponsesRunEnvelope['payload']);
const context = createResponseContext(request, record.responseId);
context.createdAt = Math.floor(record.createdAt.getTime() / 1000);
return {
...buildAggregatedResponse(context, createResponseAggregator()),
status: 'queued',
completed_at: null,
output: [],
usage: null,
background: true,
store: true,
metadata: request.metadata ?? {},
};
}
function view(record: StoredResponseRecord): Response {
return record.responseJSON
? (JSON.parse(record.responseJSON) as Response)
: buildPendingBackgroundResponse(record);
}
/** Provider-neutral Responses lifecycle; no channel, Slack, service-token or rendering dependency. */
export function createBackgroundResponseService(deps: BackgroundResponseDependencies) {
if (typeof deps.recoverTerminal !== 'function')
throw new Error('Response completion owner is unavailable');
const now = deps.now ?? Date.now;
async function readAuthorized(owner: ResponseOwner, responseId: string) {
const record = await deps.storage.read(owner, responseId);
if (!record || !(await deps.authorize(owner, record.agentId, record.sourceKeyId))) return null;
return record;
}
function hasTerminalWork(record: StoredResponseRecord): boolean {
return record.pendingTerminal != null || terminal(record.status);
}
async function canonicalView(record: StoredResponseRecord): Promise<Response | null> {
if (!hasTerminalWork(record)) return view(record);
const recovered = await deps.recoverTerminal(record);
return recovered.retirement ? null : view(recovered);
}
async function freshView(record: StoredResponseRecord): Promise<Response | null> {
const current = await deps.storage.read(record, record.responseId);
return current ? canonicalView(current) : null;
}
async function refresh(record: StoredResponseRecord): Promise<Response | null> {
if (hasTerminalWork(record)) return canonicalView(record);
const observed = await deps.observe(record);
if (!observed) return view(record);
const response = observed.response;
if (
response.id !== record.responseId ||
response.model !== record.agentId ||
!response.background ||
!response.store
)
throw new Error('Background response execution identity changed');
// An observer has no provider-drain or checkpoint proof. Only the journal may finalize it.
// Enrollment identity likewise belongs to admission, never a progress projection.
if (
terminal(response.status) ||
(observed.generationCreatedAt != null &&
observed.generationCreatedAt !== record.generationCreatedAt)
)
return freshView(record);
const result = await deps.storage.publish(record, record.responseId, record.revision, {
status:
response.required_action || (record.status === 'requires_action' && record.review != null)
? 'requires_action'
: response.status,
responseJSON: JSON.stringify(response),
});
return result.ok ? canonicalView(result.record) : freshView(record);
}
async function responseResult(record: StoredResponseRecord): Promise<BackgroundResponseResult> {
const response = await refresh(record);
return response
? { ok: true, response }
: { ok: false, status: 404, code: 'response_not_found' };
}
return {
async create(
envelope: ResponsesRunEnvelope,
idempotencyKey?: string,
sourceKeyId?: string,
): Promise<BackgroundResponseResult> {
const request = envelope.payload;
if (request.background !== true || request.store === false || request.stream === true)
return { ok: false, status: 400, code: 'invalid_background_request' };
if (idempotencyKey != null && (!idempotencyKey.trim() || idempotencyKey.length > 256))
return { ok: false, status: 400, code: 'invalid_idempotency_key' };
const owner = envelope.principal;
if (!(await deps.authorize(owner, request.model, sourceKeyId)))
return { ok: false, status: 403, code: 'permission_denied' };
const normalized = { ...request, store: true };
const requestJSON = canonicalResponseJSON(normalized);
const continuation = await deps.resolveContinuation(owner, request.previous_response_id);
const result = await deps.storage.reserve({
...owner,
...continuation,
agentId: request.model,
...(sourceKeyId == null ? {} : { sourceKeyId }),
responseId: `resp_${randomUUID().replaceAll('-', '')}`,
requestKey: idempotencyKey ?? randomUUID(),
fingerprint: createHash('sha256').update(requestJSON).digest('hex'),
requestJSON,
});
if (!result.ok) return { ok: false, status: 409, code: result.code };
let record = result.record;
if (result.replayed && !(await deps.authorize(owner, record.agentId, record.sourceKeyId)))
return { ok: false, status: 403, code: 'permission_denied' };
if (hasTerminalWork(record)) return responseResult(record);
const attached = await deps.storage.attachDelivery(
owner,
record.responseId,
deps.deliveryKey?.(record) ?? `response_${record.responseId}`,
);
if (attached.ok) {
record = attached.record;
if (!hasTerminalWork(record)) await deps.enqueue(record);
}
return hasTerminalWork(record)
? responseResult(record)
: { ok: true, response: view(record) };
},
async retrieve(owner: ResponseOwner, responseId: string): Promise<BackgroundResponseResult> {
const record = await readAuthorized(owner, responseId);
if (!record) return { ok: false, status: 404, code: 'response_not_found' };
return responseResult(record);
},
async cancel(owner: ResponseOwner, responseId: string): Promise<BackgroundResponseResult> {
const authorized = await readAuthorized(owner, responseId);
if (!authorized) return { ok: false, status: 404, code: 'response_not_found' };
if (hasTerminalWork(authorized)) return responseResult(authorized);
const record = await deps.storage.requestCancellation(owner, responseId, now());
if (!record) return { ok: false, status: 404, code: 'response_not_found' };
if (!hasTerminalWork(record)) await deps.cancel(record);
const current = await readAuthorized(owner, responseId);
if (!current) return { ok: false, status: 404, code: 'response_not_found' };
return responseResult(current);
},
};
}

View file

@ -0,0 +1,49 @@
import { checkpointerSchema } from 'librechat-data-provider';
import type { StoredResponseRecord } from '@librechat/data-schemas';
import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints';
import { deleteAgentCheckpoint } from '../checkpointer';
/** Cleanup uses captured server settings, never current config or caller-supplied collection names. */
export type ResponseCheckpointIdentity = Pick<
StoredResponseRecord,
'userId' | 'tenantId' | 'conversationId'
> & {
review?: Pick<NonNullable<StoredResponseRecord['review']>, 'checkpointConfigJSON'>;
pendingTerminal?: Pick<
NonNullable<StoredResponseRecord['pendingTerminal']>,
'checkpointNamespace' | 'checkpointConfigJSON'
>;
};
export async function cleanupResponseCheckpoint(
record: ResponseCheckpointIdentity,
checkpointNamespace: string,
): Promise<void> {
const receipt = record.pendingTerminal;
const raw = receipt?.checkpointConfigJSON ?? record.review?.checkpointConfigJSON;
if (!raw || !checkpointNamespace)
throw new Error('Response checkpoint cleanup identity unavailable');
if (
(receipt?.checkpointNamespace && receipt.checkpointNamespace !== checkpointNamespace) ||
(receipt?.checkpointConfigJSON &&
record.review?.checkpointConfigJSON &&
receipt.checkpointConfigJSON !== record.review.checkpointConfigJSON)
)
throw new Error('Response checkpoint cleanup identity changed');
if (
!checkpointNamespace.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId))
)
throw new Error('Response checkpoint owner changed');
const config = checkpointerSchema.parse(JSON.parse(raw));
if (
!config ||
config.type !== 'mongo' ||
!config.checkpointCollectionName ||
!config.checkpointWritesCollectionName
)
throw new Error('Response checkpoint must retain its durable storage settings');
await deleteAgentCheckpoint(record.conversationId, config, undefined, {
throwOnError: true,
checkpointNamespace,
});
}

View file

@ -0,0 +1,141 @@
import type { ToolCompleteEvent } from '@librechat/agents';
import type { Response as HttpResponse } from 'express';
import type { Response } from './types';
import {
buildAggregatedResponse,
createResponseAggregator,
createResponsesEventHandlers,
createAggregatorEventHandlers,
} from './service';
import { createResponseTracker } from './handlers';
const context = { responseId: 'resp_completion', model: 'agent', createdAt: 1 };
function setup(stream: boolean, announceArguments = true) {
const events: Array<{ type: string; response?: Response; call_id?: string; arguments?: string }> =
[];
const res = {
write: (chunk: string) => {
if (chunk.startsWith('data: ') && !chunk.includes('[DONE]'))
events.push(JSON.parse(chunk.slice(6)));
},
} as HttpResponse;
const streaming = createResponsesEventHandlers({
res,
tracker: createResponseTracker(),
context,
});
const aggregator = createResponseAggregator();
const handlers = stream ? streaming.handlers : createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step-a',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'call-a', name: 'protected_lookup' }],
},
});
if (announceArguments)
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: { tool_calls: [{ id: 'call-a', args: {} }] },
});
return {
handlers,
events,
finish: () => {
if (!stream) return buildAggregatedResponse(context, aggregator);
streaming.finalizeStream();
return events.find((event) => event.type === 'response.completed')!.response!;
},
};
}
function completed(output: string, id = 'call-a') {
return {
result: {
id: 'step-a',
index: 0,
type: 'tool_call',
tool_call: { id, name: 'protected_lookup', args: '{}', output, progress: 1 },
},
} satisfies { result: ToolCompleteEvent };
}
describe.each([false, true])('authoritative SDK tool completion (stream=%s)', (stream) => {
it('finishes eager calls with exact arguments and ignores fragments that arrive after completion', () => {
const f = setup(stream, false);
f.handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step-a',
delta: {
type: 'tool_calls',
tool_calls: [{ id: 'call-a', args: '{"org":' }],
},
});
const event = completed('actual-result');
event.result.tool_call.args = '{"org":"a"}';
f.handlers.on_run_step_completed.handle('on_run_step_completed', event);
const countAfterCompletion = f.events.length;
f.handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step-a',
delta: {
type: 'tool_calls',
tool_calls: [{ id: 'call-a', args: '"a"}' }],
},
});
f.handlers.on_chat_model_end.handle('on_chat_model_end', {
output: { tool_calls: [{ id: 'call-a', args: { org: 'a' } }] },
});
expect(f.events).toHaveLength(countAfterCompletion);
expect(f.finish().output.find((item) => item.type === 'function_call')).toMatchObject({
arguments: '{"org":"a"}',
status: 'completed',
});
if (stream)
expect(
f.events.find((entry) => entry.type === 'response.function_call_arguments.done'),
).toMatchObject({ call_id: 'call-a', arguments: '{"org":"a"}' });
});
it.each(['{"result":[1,2]}', ''])(
'retains the actual result including empty output: %s',
(output) => {
const f = setup(stream);
const event = completed(output);
f.handlers.on_run_step_completed.handle('on_run_step_completed', event);
f.handlers.on_run_step_completed.handle('on_run_step_completed', event);
const response = f.finish();
expect(response.output.filter((item) => item.type === 'function_call_output')).toEqual([
expect.objectContaining({ type: 'function_call_output', call_id: 'call-a', output }),
]);
expect(response.output.find((item) => item.type === 'function_call')).toMatchObject({
status: 'completed',
arguments: '{}',
});
},
);
it('does not copy unrelated completion, summary or raw artifacts into the response', () => {
const f = setup(stream);
f.handlers.on_run_step_completed.handle(
'on_run_step_completed',
completed('unrelated-secret', 'unknown-call'),
);
f.handlers.on_run_step_completed.handle('on_run_step_completed', {
result: { type: 'summary', summary: 'private-summary' },
});
f.handlers.on_run_step_completed.handle('on_run_step_completed', {
result: { tool_call: { id: 'call-a', output: { artifact: { token: 'private-token' } } } },
});
expect(f.finish().output.some((item) => item.type === 'function_call_output')).toBe(false);
expect(JSON.stringify(f.events)).not.toMatch(/unrelated-secret|private-summary|private-token/);
});
it('keeps compatibility with legacy flat completions without emitting a second result', () => {
const f = setup(stream);
f.handlers.on_tool_end.handle('on_tool_end', {
tool_call_id: 'call-a',
output: 'first-result',
});
f.handlers.on_run_step_completed.handle('on_run_step_completed', completed('first-result'));
expect(f.finish().output.filter((item) => item.type === 'function_call_output')).toEqual([
expect.objectContaining({ call_id: 'call-a', output: 'first-result' }),
]);
});
});

View file

@ -1,198 +1,296 @@
import { EModelEndpoint } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type { Response as HttpResponse } from 'express';
import type { TurnConversationRequest, ConversationWriteContext } from '~/conversations/save';
import type { PersistedReply, ReplyAnnouncement } from '~/conversations/announce';
import type { getLangfuseTraceMessageFields } from '~/langfuse/destinations';
import type { ConversationCodeEnvironmentDecision } from '~/code/decision';
import type { Response, Usage, ResponseAccounting } from './types';
import type { ForegroundResponseTerminal } from './foreground';
import type { InternalMessage } from './service';
import {
saveForegroundResponseInputs,
projectStoredResponseMessage,
requireResponseStorageWrite,
} from './foreground';
import { resolvePersistableCodeEnvironmentDecision } from '~/code/decision';
import { getConversationWriteContext } from '~/conversations/save';
import { isTemporaryTurn } from '~/conversations/retention';
import { sendResponsesErrorResponse } from './service';
import { isDeepStrictEqual } from 'node:util';
import type {
ResponseStorage,
StoredResponseRecord,
ResponseTerminalReceipt,
ResponseMutationResult,
ResponseCleanupIdentity,
} from '@librechat/data-schemas';
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { responseTenantId } from './scope';
interface CompletionRequest
extends Omit<TurnConversationRequest, 'config' | 'resolvedConversation'> {
config?: AppConfig;
resolvedConversation?:
| (ReturnType<typeof resolvePersistableCodeEnvironmentDecision> & {
conversationId?: string;
isTemporary?: boolean;
expiredAt?: Date;
})
| null;
/** A retirement result carries cleanup authority only and must never render private output. */
export interface ResponseCompletionRecord extends StoredResponseRecord {
retirement?: ResponseCleanupIdentity['retirement'];
}
type InputMessage = Parameters<
Parameters<typeof saveForegroundResponseInputs<ConversationWriteContext>>[1]['saveMessage']
>[1];
type OutputMessage = ReturnType<typeof projectStoredResponseMessage> & {
messageId: string;
conversationId: string;
isCreatedByUser: false;
sender: string;
endpoint: string;
model: string;
finish_reason: string;
tokenCount?: number;
export interface ResponseCompletion {
finish(record: StoredResponseRecord): Promise<ResponseCompletionRecord>;
stageAndFinish(
record: StoredResponseRecord,
receipt: ResponseTerminalReceipt,
): Promise<ResponseCompletionRecord>;
}
export interface ResponseCompletionDependencies {
storage: ResponseStorage;
cleanupCheckpoint: (record: StoredResponseRecord, namespace: string) => Promise<void>;
/** Required and idempotent: transient provider ownership survives until canonical publication. */
releaseExecution: (
record: StoredResponseRecord,
receipt?: ResponseTerminalReceipt,
) => Promise<void>;
}
function terminal(record: StoredResponseRecord): boolean {
return ['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status);
}
type ExecutionReleaseIdentity = Pick<
StoredResponseRecord,
| 'userId'
| 'tenantId'
| 'responseId'
| 'agentId'
| 'conversationId'
| 'generationCreatedAt'
| 'expiredAt'
| 'status'
> & {
retirement?: ResponseCleanupIdentity['retirement'];
resume?: Pick<NonNullable<StoredResponseRecord['resume']>, 'state' | 'providerExecutionId'>;
review?: Pick<NonNullable<StoredResponseRecord['review']>, 'providerExecutionId'>;
};
interface StoredReply extends PersistedReply {
conversationId: string;
messageId: string;
}
interface CompletionDependencies {
saveMessage: (
context: ConversationWriteContext,
message: InputMessage | OutputMessage,
options: { context: string },
) => Promise<StoredReply | null>;
saveConvo: (
context: ConversationWriteContext,
fields: ReturnType<typeof resolvePersistableCodeEnvironmentDecision> & {
conversationId: string;
endpoint: string;
agent_id: string;
title?: string;
model?: string;
},
options: { context: string; initialAgentId: string | null },
) => Promise<unknown>;
announceReply: (announcement: ReplyAnnouncement) => Promise<unknown>;
resolveTitle: (request: CompletionRequest, candidate: string) => string | null;
traceFields: typeof getLangfuseTraceMessageFields;
/** Release transient provider proof only after its canonical terminal result is durable. */
export function createResponseExecutionRelease(
jobs: Pick<GenerationJobManagerClass, 'getJob' | 'finishProviderTerminalPersistence'>,
): (
record: ExecutionReleaseIdentity,
receipt?: Pick<ResponseTerminalReceipt, 'providerExecutionId' | 'ownerId'>,
) => Promise<void> {
return async (record, receipt) => {
if (
!['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status) &&
!record.retirement &&
!(record.expiredAt && record.expiredAt.getTime() <= Date.now())
)
throw new Error('Response terminal publication is unavailable');
if (record.generationCreatedAt == null) return;
const job = await jobs.getJob(record.responseId);
if (!job) return;
const expectedProvider =
receipt?.providerExecutionId ??
(record.resume?.state === 'admitted'
? record.resume.providerExecutionId
: record.review?.providerExecutionId);
if (
job.createdAt !== record.generationCreatedAt ||
job.metadata.userId !== record.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
job.metadata.agent_id !== record.agentId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.responseMessageId !== record.responseId ||
job.metadata.conversationId !== record.conversationId ||
!['complete', 'error', 'aborted'].includes(job.status) ||
job.metadata.providerDrained !== true ||
!job.metadata.providerExecutionId ||
(expectedProvider != null && job.metadata.providerExecutionId !== expectedProvider) ||
(receipt != null && job.metadata.providerExecutionOwnerId !== receipt.ownerId)
)
throw new Error('Response terminal execution identity changed');
const released = await jobs.finishProviderTerminalPersistence(
record.responseId,
record.generationCreatedAt,
job.metadata.providerExecutionId,
receipt?.ownerId ?? job.metadata.providerExecutionOwnerId,
);
if (!released) throw new Error('Response terminal execution release is unavailable');
};
}
/** Own storage ordering and transport selection in the same typed completion boundary. */
export function stageForegroundResponse(
{
terminal,
request,
response,
conversationId,
agentId,
agent,
codeEnvironmentDecision,
messages,
parentMessageId,
visibleOutputTokens,
transport,
}: {
terminal: ForegroundResponseTerminal<StoredReply>;
request: CompletionRequest;
response: Response;
conversationId: string;
agentId: string;
agent: { id?: string; name?: string; model?: string };
codeEnvironmentDecision?: ConversationCodeEnvironmentDecision;
messages: (InternalMessage & { messageId: string; parentMessageId: string })[];
parentMessageId: string;
visibleOutputTokens?: number;
transport:
| { type: 'json'; response: HttpResponse }
| {
type: 'stream';
response: HttpResponse;
finalize: (usage?: Usage, accounting?: ResponseAccounting, response?: Response) => void;
fail: Parameters<ForegroundResponseTerminal<StoredReply>['stage']>[0]['fail'];
};
},
deps: CompletionDependencies,
): void {
terminal.stage({
response,
save: async (result) => {
const context = getConversationWriteContext(request);
const title = deps.resolveTitle(request, agent.name || 'Open Responses Conversation');
await requireResponseStorageWrite(
deps.saveConvo(
context,
{
conversationId,
endpoint: EModelEndpoint.agents,
agent_id: agentId,
...resolvePersistableCodeEnvironmentDecision({
conversationId,
decision: codeEnvironmentDecision,
conversation:
request.resolvedConversation?.conversationId != null
? {
conversationId: request.resolvedConversation.conversationId,
codeEnvironmentMode: request.resolvedConversation.codeEnvironmentMode,
codeWorkspaces: request.resolvedConversation.codeWorkspaces,
}
: undefined,
}),
...(title != null && { title }),
model: agent.model,
},
{
context: 'Responses API - save conversation',
initialAgentId: agent.id === agentId ? agentId : null,
},
),
{ conversationId },
);
await saveForegroundResponseInputs(
{ context, conversationId, messages, agentId },
{ saveMessage: deps.saveMessage },
);
const traceFields = await deps.traceFields(request.config, result.id, {
isTemporary: isTemporaryTurn(request),
});
return requireResponseStorageWrite(
deps.saveMessage(
context,
{
messageId: result.id,
conversationId,
...projectStoredResponseMessage(result, parentMessageId),
isCreatedByUser: false,
...traceFields,
sender: 'Agent',
endpoint: EModelEndpoint.agents,
model: agentId,
finish_reason: result.status === 'completed' ? 'stop' : result.status,
tokenCount: visibleOutputTokens ?? result.usage?.output_tokens,
},
{ context: 'Responses API - save assistant response' },
),
{ conversationId, messageId: result.id },
).then((saved) => {
if (!saved) throw new Error('Response write returned no reply');
return saved;
});
},
afterStore: async (reply) => {
await deps.announceReply({
userId: request.user?.id,
conversationId,
reply,
context: 'Responses API - announce stored reply',
});
},
fail: (failure) => {
if (transport.type === 'stream') return transport.fail(failure);
sendResponsesErrorResponse(
transport.response,
failure.status,
failure.message,
'server_error',
failure.code,
);
},
publish: (result) => {
if (transport.type === 'json') {
transport.response.json(result);
return;
}
transport.finalize(result.usage ?? undefined, result.accounting, result);
transport.response.end();
},
});
function assertReceipt(record: StoredResponseRecord, receipt: ResponseTerminalReceipt): void {
let response: unknown;
try {
response = JSON.parse(receipt.responseJSON);
} catch {
throw new Error('Response terminal receipt is invalid');
}
if (
receipt.generationCreatedAt !== record.generationCreatedAt ||
!response ||
typeof response !== 'object' ||
!('id' in response) ||
response.id !== record.responseId ||
!('model' in response) ||
response.model !== record.agentId ||
!('status' in response) ||
response.status !== receipt.status ||
!('object' in response) ||
response.object !== 'response' ||
!('output' in response) ||
!Array.isArray(response.output) ||
(receipt.checkpointConfigJSON != null && !receipt.checkpointNamespace) ||
!('background' in response) ||
response.background !== true ||
!('store' in response) ||
response.store !== true
)
throw new Error('Response terminal receipt identity changed');
}
function sameExecution(
previous: StoredResponseRecord,
current: StoredResponseRecord,
receipt: ResponseTerminalReceipt,
): boolean {
const provider =
current.resume?.state === 'admitted'
? current.resume.providerExecutionId
: current.review?.providerExecutionId;
return (
current.userId === previous.userId &&
responseTenantId(current.tenantId) === responseTenantId(previous.tenantId) &&
current.responseId === previous.responseId &&
current.agentId === previous.agentId &&
current.sourceKeyId === previous.sourceKeyId &&
current.conversationId === previous.conversationId &&
current.generationCreatedAt === receipt.generationCreatedAt &&
(provider == null || provider === receipt.providerExecutionId) &&
isDeepStrictEqual(current.review, previous.review) &&
isDeepStrictEqual(current.resume, previous.resume)
);
}
/** Persist a post-drain result before cleanup, then retry only cleanup/publication, never execution. */
export function createResponseCompletion(deps: ResponseCompletionDependencies): ResponseCompletion {
if (typeof deps.cleanupCheckpoint !== 'function' || typeof deps.releaseExecution !== 'function')
throw new Error('Response completion cleanup and release owners are unavailable');
async function discardRetired(record: StoredResponseRecord, receipt: ResponseTerminalReceipt) {
const identity = await deps.storage.readCleanupIdentity(record, record.responseId, {
...record,
checkpointNamespace: receipt.checkpointNamespace,
checkpointConfigJSON: receipt.checkpointConfigJSON,
});
if (
!identity?.retirement &&
!(identity?.expiredAt && identity.expiredAt.getTime() <= Date.now())
)
return null;
const retired: ResponseCompletionRecord = {
...record,
...identity,
retirement: identity.retirement ?? 'expired',
};
const namespace = identity.checkpointNamespace ?? receipt.checkpointNamespace;
if (identity.requiresCheckpointCleanup && !namespace)
throw new Error('Response checkpoint cleanup identity unavailable');
if (namespace)
await deps.cleanupCheckpoint(
{
...retired,
review: undefined,
pendingTerminal: {
...receipt,
checkpointNamespace: namespace,
checkpointConfigJSON: identity.checkpointConfigJSON ?? receipt.checkpointConfigJSON,
},
},
namespace,
);
await deps.releaseExecution(retired, receipt);
return {
...retired,
requestJSON: '',
responseJSON: undefined,
review: undefined,
resume: undefined,
pendingTerminal: undefined,
status: receipt.status,
};
}
async function finish(record: StoredResponseRecord): Promise<ResponseCompletionRecord> {
const receipt = record.pendingTerminal;
if (!receipt) {
if (terminal(record)) await deps.releaseExecution(record);
return record;
}
assertReceipt(record, receipt);
if (receipt.checkpointNamespace)
await deps.cleanupCheckpoint(record, receipt.checkpointNamespace);
let current: StoredResponseRecord | null;
try {
const result = await deps.storage.finishTerminal(
record,
record.responseId,
record.revision,
receipt,
);
current = result.ok ? result.record : await deps.storage.read(record, record.responseId);
} catch (error) {
current = await deps.storage.read(record, record.responseId);
if (!current) {
const expired = await discardRetired(record, receipt);
if (expired) return expired;
}
if (
!current ||
current.status !== receipt.status ||
current.generationCreatedAt !== receipt.generationCreatedAt ||
current.responseJSON !== receipt.responseJSON ||
current.pendingTerminal
)
throw error;
}
if (!current) {
const expired = await discardRetired(record, receipt);
if (expired) return expired;
throw new Error('Response terminal journal is unavailable');
}
if (terminal(current)) await deps.releaseExecution(current, receipt);
return current;
}
return {
finish,
async stageAndFinish(
record: StoredResponseRecord,
receipt: ResponseTerminalReceipt,
): Promise<ResponseCompletionRecord> {
assertReceipt(record, receipt);
if (record.expiredAt && record.expiredAt.getTime() <= Date.now()) {
const retired = await discardRetired(record, receipt);
if (!retired) throw new Error('Response retirement identity unavailable');
return retired;
}
let staged: ResponseMutationResult;
try {
staged = await deps.storage.stageTerminal(
record,
record.responseId,
{ execution: record },
receipt,
);
} catch (error) {
const current = await deps.storage.read(record, record.responseId);
if (!current) {
const expired = await discardRetired(record, receipt);
if (expired) return expired;
}
const saved = current?.pendingTerminal;
if (
!current ||
!saved ||
saved.generationCreatedAt !== receipt.generationCreatedAt ||
saved.providerExecutionId !== receipt.providerExecutionId ||
saved.ownerId !== receipt.ownerId ||
saved.responseJSON !== receipt.responseJSON ||
saved.status !== receipt.status ||
saved.checkpointNamespace !== receipt.checkpointNamespace ||
saved.checkpointConfigJSON !== receipt.checkpointConfigJSON
)
throw error;
return finish(current);
}
if (staged.ok) return finish(staged.record);
const current = await deps.storage.read(record, record.responseId);
if (!current) {
const expired = await discardRetired(record, receipt);
if (expired) return expired;
}
if (!current || !sameExecution(record, current, receipt))
throw new Error('Response terminal receipt publication conflicted');
if (terminal(current)) return finish(current);
throw new Error('Response terminal receipt publication conflicted');
},
};
}

View file

@ -1,14 +1,23 @@
import { ContentTypes, EModelEndpoint } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type { Response as HttpResponse } from 'express';
import type { RequestHandler } from 'express';
import type { TurnConversationRequest, ConversationWriteContext } from '~/conversations/save';
import type { Response, ResponseContext, Usage, ResponseAccounting } from './types';
import type { PersistedReply, ReplyAnnouncement } from '~/conversations/announce';
import type { getLangfuseTraceMessageFields } from '~/langfuse/destinations';
import type { ConversationCodeEnvironmentDecision } from '~/code/decision';
import type { AgentExecutionEnrollment } from '../remote/lifecycle';
import type { StoredResponseMessage } from './persistence';
import type { Response, ResponseContext } from './types';
import type { InternalMessage } from './service';
import {
ResponsePersistenceError,
selectResponseHistory,
projectResponseMessage,
} from './persistence';
import { resolvePersistableCodeEnvironmentDecision } from '~/code/decision';
import { getConversationWriteContext } from '~/conversations/save';
import { isTemporaryTurn } from '~/conversations/retention';
import { sendResponsesErrorResponse } from './service';
import { getSafeErrorMetadata } from '~/utils/errors';
@ -347,3 +356,182 @@ export function createForegroundResponseTerminal<Saved>({
},
};
}
interface CompletionRequest
extends Omit<TurnConversationRequest, 'config' | 'resolvedConversation'> {
config?: AppConfig;
resolvedConversation?:
| (ReturnType<typeof resolvePersistableCodeEnvironmentDecision> & {
conversationId?: string;
isTemporary?: boolean;
expiredAt?: Date;
})
| null;
}
type InputMessage = Parameters<
Parameters<typeof saveForegroundResponseInputs<ConversationWriteContext>>[1]['saveMessage']
>[1];
type OutputMessage = ReturnType<typeof projectStoredResponseMessage> & {
messageId: string;
conversationId: string;
isCreatedByUser: false;
sender: string;
endpoint: string;
model: string;
finish_reason: string;
tokenCount?: number;
};
interface StoredReply extends PersistedReply {
conversationId: string;
messageId: string;
}
interface CompletionDependencies {
saveMessage: (
context: ConversationWriteContext,
message: InputMessage | OutputMessage,
options: { context: string },
) => Promise<StoredReply | null>;
saveConvo: (
context: ConversationWriteContext,
fields: ReturnType<typeof resolvePersistableCodeEnvironmentDecision> & {
conversationId: string;
endpoint: string;
agent_id: string;
title?: string;
model?: string;
},
options: { context: string; initialAgentId: string | null },
) => Promise<unknown>;
announceReply: (announcement: ReplyAnnouncement) => Promise<unknown>;
resolveTitle: (request: CompletionRequest, candidate: string) => string | null;
traceFields: typeof getLangfuseTraceMessageFields;
}
/** Own storage ordering and transport selection in the same typed completion boundary. */
export function stageForegroundResponse(
{
terminal,
request,
response,
conversationId,
agentId,
agent,
codeEnvironmentDecision,
messages,
parentMessageId,
visibleOutputTokens,
transport,
}: {
terminal: ForegroundResponseTerminal<StoredReply>;
request: CompletionRequest;
response: Response;
conversationId: string;
agentId: string;
agent: { id?: string; name?: string; model?: string };
codeEnvironmentDecision?: ConversationCodeEnvironmentDecision;
messages: (InternalMessage & { messageId: string; parentMessageId: string })[];
parentMessageId: string;
visibleOutputTokens?: number;
transport:
| { type: 'json'; response: HttpResponse }
| {
type: 'stream';
response: HttpResponse;
finalize: (usage?: Usage, accounting?: ResponseAccounting, response?: Response) => void;
fail: Parameters<ForegroundResponseTerminal<StoredReply>['stage']>[0]['fail'];
};
},
deps: CompletionDependencies,
): void {
terminal.stage({
response,
save: async (result) => {
const context = getConversationWriteContext(request);
const title = deps.resolveTitle(request, agent.name || 'Open Responses Conversation');
await requireResponseStorageWrite(
deps.saveConvo(
context,
{
conversationId,
endpoint: EModelEndpoint.agents,
agent_id: agentId,
...resolvePersistableCodeEnvironmentDecision({
conversationId,
decision: codeEnvironmentDecision,
conversation:
request.resolvedConversation?.conversationId != null
? {
conversationId: request.resolvedConversation.conversationId,
codeEnvironmentMode: request.resolvedConversation.codeEnvironmentMode,
codeWorkspaces: request.resolvedConversation.codeWorkspaces,
}
: undefined,
}),
...(title != null && { title }),
model: agent.model,
},
{
context: 'Responses API - save conversation',
initialAgentId: agent.id === agentId ? agentId : null,
},
),
{ conversationId },
);
await saveForegroundResponseInputs(
{ context, conversationId, messages, agentId },
{ saveMessage: deps.saveMessage },
);
const traceFields = await deps.traceFields(request.config, result.id, {
isTemporary: isTemporaryTurn(request),
});
return requireResponseStorageWrite(
deps.saveMessage(
context,
{
messageId: result.id,
conversationId,
...projectStoredResponseMessage(result, parentMessageId),
isCreatedByUser: false,
...traceFields,
sender: 'Agent',
endpoint: EModelEndpoint.agents,
model: agentId,
finish_reason: result.status === 'completed' ? 'stop' : result.status,
tokenCount: visibleOutputTokens ?? result.usage?.output_tokens,
},
{ context: 'Responses API - save assistant response' },
),
{ conversationId, messageId: result.id },
).then((saved) => {
if (!saved) throw new Error('Response write returned no reply');
return saved;
});
},
afterStore: async (reply) => {
await deps.announceReply({
userId: request.user?.id,
conversationId,
reply,
context: 'Responses API - announce stored reply',
});
},
fail: (failure) => {
if (transport.type === 'stream') return transport.fail(failure);
sendResponsesErrorResponse(
transport.response,
failure.status,
failure.message,
'server_error',
failure.code,
);
},
publish: (result) => {
if (transport.type === 'json') {
transport.response.json(result);
return;
}
transport.finalize(result.usage ?? undefined, result.accounting, result);
transport.response.end();
},
});
}

View file

@ -13,6 +13,17 @@ export {
projectResponseMessage,
resolveResponseContinuation,
} from './persistence';
export { createBackgroundResponseService } from './background';
export { createBackgroundResponseAuthority } from './authority';
export { createBackgroundResponseQueue, backgroundResponseDeliveryKey } from './queue';
export {
createBackgroundResponseWorkerAuth,
mintBackgroundResponseWorkerToken,
mintResponseResumeWorkerToken,
} from './worker';
export type { BackgroundResponseDependencies, BackgroundResponseResult } from './background';
export {
createResponseAccountingCollector,
@ -214,5 +225,4 @@ export { createClientToolHandoff, type ClientToolHandoff } from './clientTools';
export * from './foreground';
export * from './stored';
export * from './completion';
export type { ResponseCompletion, ResponseCompletionRecord } from './completion';

View file

@ -0,0 +1,449 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createModels,
createMethods,
createResponseStorage,
tenantStorage,
runAsSystem,
} from '@librechat/data-schemas';
import type { ResponseStorage, StoredResponseRecord, ResponseOwner } from '@librechat/data-schemas';
import type { AgentTriggerService, AgentTriggerDeliveryPersistence } from '../triggers/service';
import {
createBackgroundResponseQueue,
createBackgroundResponseEnvelope,
createResponseResumeQueue,
createResponseResumeEnvelope,
BACKGROUND_RESPONSE_RESUME_SOURCE,
} from './queue';
import { createAgentTriggerService } from '../triggers/service';
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
let methods: ReturnType<typeof createMethods>;
let service: AgentTriggerService | undefined;
const owner = { userId: '000000000000000000000001', tenantId: 'response-queue-test' };
const scope = <T>(work: () => Promise<T>) => tenantStorage.run({ tenantId: owner.tenantId }, work);
function persistence(): AgentTriggerDeliveryPersistence {
return {
...methods,
getAgentTriggerDeliveryBatch: (delivery) =>
methods.getAgentTriggerDeliveryBatch({
id: delivery.id,
batchMemberIds: delivery.batchMemberIds?.map(
(id) => new mongoose.Types.ObjectId(String(id)),
),
}),
};
}
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 1 },
instanceOpts: [{ args: ['--nounixsocket'] }],
});
await mongoose.connect(mongo.getUri(), { autoIndex: false });
createModels(mongoose);
methods = createMethods(mongoose);
storage = createResponseStorage(mongoose);
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
afterEach(async () => {
await service?.stop();
service = undefined;
});
beforeEach(async () => {
await runAsSystem(async () => {
await Promise.all(
[
'StoredResponse',
'ResponseScope',
'AgentTriggerDelivery',
'AgentTriggerLaneSequence',
'AgentTriggerUserPurge',
].map((name) => mongoose.models[name].deleteMany({})),
);
});
});
async function reserve(principal: ResponseOwner = owner): Promise<StoredResponseRecord> {
const result = await storage.reserve({
...principal,
responseId: 'resp_queue',
requestKey: 'key',
fingerprint: 'request',
agentId: 'agent',
sourceKeyId: 'original-api-key',
requestJSON: JSON.stringify({
model: 'agent',
input: 'Original input',
instructions: 'Preserve instructions',
background: true,
store: true,
}),
conversationId: 'conversation',
parentMessageId: 'root',
});
if (!result.ok) throw new Error('Reservation failed');
return result.record;
}
it('executes queued Responses through its registered runtime, never through chat HTTP', () =>
scope(async () => {
const executed: StoredResponseRecord[] = [];
const fetcher = jest.fn(async () => {
throw new Error('Chat transport must not run');
});
const queue = createBackgroundResponseQueue({
storage,
enqueue: (...args) => service!.enqueue(...args),
authorize: async (record) => record.sourceKeyId === 'original-api-key',
execute: async (record) => {
executed.push(record);
return { status: 'started', generationCreatedAt: 100 };
},
});
service = createAgentTriggerService({
methods: persistence(),
sourceDispatchers: queue.sourceDispatchers,
fetch: fetcher,
deliveryOptions: { tickMs: 10 },
});
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } });
const reserved = await reserve();
const attached = await storage.attachDelivery(
owner,
reserved.responseId,
queue.deliveryKey(reserved),
);
if (!attached.ok) throw new Error('Attach failed');
await queue.enqueue(attached.record);
for (let attempt = 0; attempt < 100 && executed.length === 0; attempt++) {
await new Promise((resolve) => setTimeout(resolve, 10));
}
expect(executed).toHaveLength(1);
expect(JSON.parse(executed[0].requestJSON)).toMatchObject({
instructions: 'Preserve instructions',
input: 'Original input',
});
expect(fetcher).not.toHaveBeenCalled();
const delivery = await service.getDelivery(attached.record.deliveryKey!);
expect(delivery?.requiredWorkerCapability).toBe('responses_background_v2');
expect(JSON.stringify(delivery?.envelope)).not.toContain('original-api-key');
expect(JSON.stringify(delivery?.envelope)).not.toContain('Original input');
}));
it('does not let a queue worker without the Responses capability claim the response', () =>
scope(async () => {
const fetcher = jest.fn(async () => {
throw new Error('Old worker must not run');
});
service = createAgentTriggerService({
methods: persistence(),
fetch: fetcher,
deliveryOptions: { tickMs: 10 },
});
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } });
const queue = createBackgroundResponseQueue({
storage,
enqueue: (...args) => service!.enqueue(...args),
authorize: async () => true,
execute: async () => {
throw new Error('Not installed');
},
});
const record = await reserve();
const attached = await storage.attachDelivery(
owner,
record.responseId,
queue.deliveryKey(record),
);
if (!attached.ok) throw new Error('Attach failed');
await queue.enqueue(attached.record);
for (const workerCapabilities of [[], ['responses_background_v1']]) {
const claimed = await runAsSystem(async () =>
methods.claimNextAgentTriggerDelivery({
workerId: 'old-worker',
claimToken: 'old-token',
now: new Date(),
leaseUntil: new Date(Date.now() + 1000),
workerCapabilities,
}),
);
expect(claimed).toBeNull();
}
expect(fetcher).not.toHaveBeenCalled();
}));
it('rejects a tampered source envelope and revoked original authority before execution', () =>
scope(async () => {
const execute = jest.fn(async () => ({ status: 'started' as const }));
let authorized = true;
const queue = createBackgroundResponseQueue({
storage,
enqueue: async () => {
throw new Error('Not used');
},
authorize: async () => authorized,
execute,
});
const record = await reserve();
await storage.attachDelivery(owner, record.responseId, queue.deliveryKey(record));
const dispatch = queue.sourceDispatchers.get('responses')!.dispatch;
const expected = createBackgroundResponseEnvelope(record);
await expect(dispatch({ ...expected, input: 'tampered' })).rejects.toMatchObject({
code: 'response_authority_changed',
});
authorized = false;
await expect(dispatch(expected)).rejects.toMatchObject({ code: 'response_authority_changed' });
expect(execute).not.toHaveBeenCalled();
}));
async function reserveDecision(): Promise<StoredResponseRecord> {
await reserve();
await storage.publish(owner, 'resp_queue', 0, {
status: 'in_progress',
generationCreatedAt: 100,
responseJSON: '{}',
});
const paused = await storage.publishReview(
owner,
'resp_queue',
1,
100,
{
actionId: 'action-one',
providerExecutionId: 'provider-one',
expiresAt: Date.now() + 60_000,
partialResponseJSON: '{"private":"unfinished output"}',
},
'{}',
);
if (!paused.ok) throw new Error('Pause failed');
const result = await storage.reserveResume(
owner,
'resp_queue',
{
generationCreatedAt: 100,
actionId: 'action-one',
intentId: 'intent-one',
previousProviderExecutionId: 'provider-one',
providerExecutionId: 'provider-two',
fingerprint: 'decision-one',
resolutionJSON: '{"answer":"private owner answer"}',
},
Date.now(),
);
if (!result.ok) throw new Error('Decision reservation failed');
return result.record;
}
it('deduplicates saved decisions in the existing queue and excludes old Responses workers', () =>
scope(async () => {
const fetcher = jest.fn(async () => {
throw new Error('Chat must not run');
});
service = createAgentTriggerService({
methods: persistence(),
fetch: fetcher,
deliveryOptions: { tickMs: 10 },
});
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } });
const queue = createResponseResumeQueue({
storage,
enqueue: (...args) => service!.enqueue(...args),
authorize: async () => true,
resume: async () => {
throw new Error('Resume host is not installed');
},
});
const record = await reserveDecision();
await Promise.all(Array.from({ length: 8 }, () => queue.enqueue(record)));
const delivery = await service.getDelivery(queue.deliveryKey(record));
expect(delivery?.requiredWorkerCapability).toBe('responses_resume_v1');
expect(await mongoose.models.AgentTriggerDelivery.countDocuments({})).toBe(1);
const serialized = JSON.stringify(delivery?.envelope);
for (const secret of [
'private owner answer',
'unfinished output',
'original-api-key',
'Original input',
])
expect(serialized).not.toContain(secret);
for (const workerCapabilities of [
[],
['responses_background_v1'],
['responses_background_v2'],
]) {
expect(
await runAsSystem(() =>
methods.claimNextAgentTriggerDelivery({
workerId: 'old-worker',
claimToken: 'old-token',
now: new Date(),
leaseUntil: new Date(Date.now() + 1000),
workerCapabilities,
}),
),
).toBeNull();
}
const claimed = await runAsSystem(() =>
methods.claimNextAgentTriggerDelivery({
workerId: 'resume-worker',
claimToken: 'resume-token',
now: new Date(),
leaseUntil: new Date(Date.now() + 1000),
workerCapabilities: ['responses_resume_v1'],
}),
);
expect(claimed).not.toBeNull();
expect(fetcher).not.toHaveBeenCalled();
}));
it('rechecks exact saved decision and source grant before dispatching the resume checkpoint', () =>
scope(async () => {
const execute = jest.fn(async () => ({ status: 'resumed' as const, generationCreatedAt: 100 }));
let authorized = true;
const queue = createResponseResumeQueue({
storage,
enqueue: async () => {
throw new Error('Not used');
},
authorize: async () => authorized,
resume: execute,
});
const record = await reserveDecision();
const envelope = createResponseResumeEnvelope(record);
const dispatch = queue.sourceDispatchers.get(BACKGROUND_RESPONSE_RESUME_SOURCE)!.dispatch;
await expect(dispatch({ ...envelope, input: 'tampered' })).rejects.toMatchObject({
code: 'response_resume_authority_changed',
});
await expect(
dispatch({ ...envelope, event: { ...envelope.event, id: 'other-intent' } }),
).rejects.toMatchObject({ code: 'response_resume_authority_changed' });
authorized = false;
await expect(dispatch(envelope)).rejects.toMatchObject({
code: 'response_resume_authority_changed',
});
expect(execute).not.toHaveBeenCalled();
authorized = true;
expect(await dispatch(envelope)).toMatchObject({
status: 'resumed',
streamId: record.responseId,
generationCreatedAt: 100,
});
expect(execute).toHaveBeenCalledTimes(1);
expect(execute).toHaveBeenCalledWith(
expect.objectContaining({ resume: expect.objectContaining({ intentId: 'intent-one' }) }),
undefined,
);
}));
it('never starts a cancelled decision and does not confuse queue settlement with provider drain', () =>
scope(async () => {
const execute = jest.fn(async () => ({ status: 'resumed' as const }));
const queue = createResponseResumeQueue({
storage,
enqueue: async () => {
throw new Error('Not used');
},
authorize: async () => true,
resume: execute,
});
const record = await reserveDecision();
await storage.requestCancellation(owner, record.responseId, Date.now());
const dispatch = queue.sourceDispatchers.get(BACKGROUND_RESPONSE_RESUME_SOURCE)!.dispatch;
expect(await dispatch(createResponseResumeEnvelope(record))).toMatchObject({
status: 'settled',
});
expect(execute).not.toHaveBeenCalled();
expect((await storage.read(owner, record.responseId))?.status).toBe('requires_action');
}));
it('rejects an old delivery after a newer provider segment has paused for a different action', () =>
scope(async () => {
const execute = jest.fn(async () => ({ status: 'resumed' as const }));
const queue = createResponseResumeQueue({
storage,
enqueue: async () => {
throw new Error('Not used');
},
authorize: async () => true,
resume: execute,
});
const record = await reserveDecision();
const admitted = await storage.admitResume(
owner,
record.responseId,
'intent-one',
100,
'provider-two',
'{}',
Date.now(),
);
if (!admitted.ok) throw new Error('Admission failed');
await storage.publishReview(
owner,
record.responseId,
admitted.record.revision,
100,
{
actionId: 'action-two',
providerExecutionId: 'provider-two',
expiresAt: Date.now() + 60_000,
partialResponseJSON: '{}',
},
'{}',
);
const dispatch = queue.sourceDispatchers.get(BACKGROUND_RESPONSE_RESUME_SOURCE)!.dispatch;
await expect(dispatch(createResponseResumeEnvelope(record))).rejects.toMatchObject({
code: 'response_resume_authority_changed',
});
expect(execute).not.toHaveBeenCalled();
}));
it.each([null, ''])(
'dispatches a legacy tenantless stored response %p through the actual trigger queue',
async (tenantId) => {
const principal = { userId: owner.userId };
await tenantStorage.run(principal, async () => {
const execute = jest.fn().mockResolvedValue({ status: 'started', generationCreatedAt: 100 });
const queue = createBackgroundResponseQueue({
storage,
enqueue: (...args) => service!.enqueue(...args),
authorize: async () => true,
execute,
});
service = createAgentTriggerService({
methods: persistence(),
sourceDispatchers: queue.sourceDispatchers,
deliveryOptions: { tickMs: 10 },
});
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } });
const saved = await reserve(principal);
const key = queue.deliveryKey(saved);
await mongoose.models.StoredResponse.collection.updateOne(
{ responseId: saved.responseId },
{ $set: { tenantId } },
);
const legacy = await storage.read(principal, saved.responseId);
if (!legacy) throw new Error('Legacy fixture missing');
expect(queue.deliveryKey(legacy)).toBe(key);
const attached = await storage.attachDelivery(principal, legacy.responseId, key);
if (!attached.ok) throw new Error('Attach failed');
await queue.enqueue(attached.record);
for (let attempt = 0; attempt < 100 && execute.mock.calls.length === 0; attempt++)
await new Promise((resolve) => setTimeout(resolve, 10));
expect(execute).toHaveBeenCalledTimes(1);
expect(execute.mock.calls[0][0]).toMatchObject({
userId: principal.userId,
tenantId,
responseId: saved.responseId,
});
});
},
);

View file

@ -0,0 +1,240 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1,
} from '@librechat/data-schemas';
import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { AgentTriggerExecutionHost } from '../triggers/host';
import type { AgentTriggerEnvelope } from '../triggers/envelope';
import type { AgentTriggerService } from '../triggers/service';
import {
createAgentTriggerEnvelope,
getAgentTriggerIdempotencyKey,
parseAgentTriggerEnvelope,
} from '../triggers/envelope';
import { prepareAgentTriggerDelivery } from '../triggers/delivery';
import { responseTenantId, isResponseTenantId } from './scope';
import { AgentTriggerExecutionError } from '../triggers/host';
export const BACKGROUND_RESPONSE_SOURCE = 'responses';
export const BACKGROUND_RESPONSE_RESUME_SOURCE = 'responses.resume';
/** A saved decision selects this identity once; HTTP/queue retries reuse it verbatim. */
export function createResponseResumeEnvelope(record: StoredResponseRecord): AgentTriggerEnvelope {
const intent = record.resume;
if (!intent || record.generationCreatedAt == null || !isResponseTenantId(record.tenantId))
throw new Error('Response resume intent is unavailable');
const identity = `${record.responseId}:resume:${intent.intentId}`;
return createAgentTriggerEnvelope({
mode: 'fire',
requestId: identity,
deliveryId: identity,
receivedAt: intent.reservedAt,
principal: { id: record.userId, tenantId: responseTenantId(record.tenantId) },
target: { agentId: record.agentId },
run: { conversationId: record.conversationId },
event: {
id: intent.intentId,
type: 'response.resume',
occurredAt: intent.reservedAt,
source: { id: record.responseId, type: BACKGROUND_RESPONSE_RESUME_SOURCE },
},
input: 'Resume the saved Responses checkpoint.',
});
}
/** Install only when this host implements same-checkpoint resume, never as a chat fallback. */
export function createResponseResumeQueue(
deps: Omit<BackgroundResponseQueueDependencies, 'execute'> & {
/** Admit only the saved intent's provider segment within its original generation epoch. */
resume: BackgroundResponseQueueDependencies['execute'];
},
): BackgroundResponseQueue {
const deliveryKey = (record: StoredResponseRecord) =>
getAgentTriggerIdempotencyKey(createResponseResumeEnvelope(record));
const dispatch: AgentTriggerExecutionHost['dispatch'] = async (input, options) => {
const envelope = parseAgentTriggerEnvelope(input);
const invalid = () =>
new AgentTriggerExecutionError('Response resume authority changed', {
mode: envelope.mode,
certainty: 'definite',
retryable: false,
code: 'response_resume_authority_changed',
status: 403,
});
if (
envelope.event.source.type !== BACKGROUND_RESPONSE_RESUME_SOURCE ||
envelope.mode !== 'fire'
)
throw invalid();
const record = await deps.storage.read(envelope.principal, envelope.event.source.id);
if (!record?.resume || !(await deps.authorize(record))) throw invalid();
if (
prepareAgentTriggerDelivery(envelope).fingerprint !==
prepareAgentTriggerDelivery(createResponseResumeEnvelope(record)).fingerprint
)
throw invalid();
const identity = {
mode: 'fire' as const,
conversationId: record.conversationId,
streamId: record.responseId,
responseMessageId: record.responseId,
generationCreatedAt: record.generationCreatedAt,
};
// A persisted decision receipt survives cancellation; it must never restart the provider.
if (
record.cancellationRequestedAt != null ||
['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status)
)
return { ...identity, status: 'settled' };
if (!['requires_action', 'in_progress'].includes(record.status)) throw invalid();
const execution = await deps.resume(record, options?.signal);
return { ...identity, ...execution };
};
return {
deliveryKey,
sourceDispatchers: new Map([
[
BACKGROUND_RESPONSE_RESUME_SOURCE,
{
workerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1,
dispatch,
},
],
]),
async enqueue(record) {
const envelope = createResponseResumeEnvelope(record);
const receipt = await deps.enqueue(envelope, {
orderingKey: `responses:${responseTenantId(record.tenantId) ?? '_'}:${record.userId}:${record.conversationId}`,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1,
});
if (receipt.deliveryKey !== getAgentTriggerIdempotencyKey(envelope))
throw new Error('Response resume queue receipt identity changed');
},
};
}
/** The queue carries an immutable response reference, never credentials or a second request body. */
export function createBackgroundResponseEnvelope(
record: StoredResponseRecord,
): AgentTriggerEnvelope {
if (!isResponseTenantId(record.tenantId))
throw new Error('Response tenant authority unavailable');
return createAgentTriggerEnvelope({
mode: 'fire',
requestId: record.responseId,
deliveryId: record.responseId,
receivedAt: record.createdAt.getTime(),
principal: { id: record.userId, tenantId: responseTenantId(record.tenantId) },
target: { agentId: record.agentId },
run: { conversationId: record.conversationId },
event: {
id: record.responseId,
type: 'response.create',
occurredAt: record.createdAt.getTime(),
source: { id: record.responseId, type: BACKGROUND_RESPONSE_SOURCE },
},
input: 'Execute the saved Responses request.',
});
}
export function backgroundResponseDeliveryKey(record: StoredResponseRecord): string {
return getAgentTriggerIdempotencyKey(createBackgroundResponseEnvelope(record));
}
export interface BackgroundResponseQueueDependencies {
storage: ResponseStorage;
enqueue: AgentTriggerService['enqueue'];
authorize: (record: StoredResponseRecord) => Promise<boolean>;
workerCapability?: string;
/** Execute the persisted Responses payload with its exact stable identity.
* Admission MUST be idempotent: a repeated delivery attaches rather than starting a second run.
* The runtime also owns cancellation-before-start and already terminal reconciliation. */
execute: (
record: StoredResponseRecord,
signal?: AbortSignal,
) => Promise<{
status: 'started' | 'resumed' | 'settled';
generationCreatedAt?: number;
}>;
}
export interface BackgroundResponseQueue {
deliveryKey: (record: StoredResponseRecord) => string;
sourceDispatchers: ReadonlyMap<
string,
{
workerCapability: string;
dispatch: AgentTriggerExecutionHost['dispatch'];
}
>;
enqueue: (record: StoredResponseRecord) => Promise<void>;
}
export function createBackgroundResponseQueue(
deps: BackgroundResponseQueueDependencies,
): BackgroundResponseQueue {
const dispatch: AgentTriggerExecutionHost['dispatch'] = async (input, options) => {
const envelope = parseAgentTriggerEnvelope(input);
const invalid = () =>
new AgentTriggerExecutionError('Background response authority changed', {
mode: envelope.mode,
certainty: 'definite',
retryable: false,
code: 'response_authority_changed',
status: 403,
});
if (envelope.event.source.type !== BACKGROUND_RESPONSE_SOURCE || envelope.mode !== 'fire')
throw invalid();
const record = await deps.storage.read(envelope.principal, envelope.event.source.id);
if (!record || !(await deps.authorize(record))) throw invalid();
const expected = createBackgroundResponseEnvelope(record);
if (
prepareAgentTriggerDelivery(envelope).fingerprint !==
prepareAgentTriggerDelivery(expected).fingerprint ||
record.deliveryKey !== backgroundResponseDeliveryKey(record)
)
throw invalid();
if (['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status)) {
return {
mode: 'fire',
status: 'settled',
conversationId: record.conversationId,
streamId: record.responseId,
responseMessageId: record.responseId,
generationCreatedAt: record.generationCreatedAt,
};
}
const execution = await deps.execute(record, options?.signal);
return {
mode: 'fire',
...execution,
conversationId: record.conversationId,
streamId: record.responseId,
responseMessageId: record.responseId,
};
};
return {
deliveryKey: backgroundResponseDeliveryKey,
sourceDispatchers: new Map([
[
BACKGROUND_RESPONSE_SOURCE,
{
workerCapability: deps.workerCapability ?? AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_V2,
dispatch,
},
],
]),
async enqueue(record: StoredResponseRecord) {
if (record.deliveryKey !== backgroundResponseDeliveryKey(record))
throw new Error('Response queue identity changed');
const receipt = await deps.enqueue(createBackgroundResponseEnvelope(record), {
orderingKey: `responses:${responseTenantId(record.tenantId) ?? '_'}:${record.userId}:${record.conversationId}`,
requiredWorkerCapability:
deps.workerCapability ?? AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_V2,
});
if (receipt.deliveryKey !== record.deliveryKey)
throw new Error('Response queue receipt identity changed');
},
};
}

View file

@ -0,0 +1,630 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createResponseStorage,
createModels,
createMethods,
tenantStorage,
runAsSystem,
} from '@librechat/data-schemas';
import type {
ResponseStorage,
StoredResponseRecord,
ResponseTerminalReceipt,
} from '@librechat/data-schemas';
import { createBackgroundResponseService, buildPendingBackgroundResponse } from './background';
import { createCheckpointNamespace } from '~/stream/checkpoints';
import { cleanupResponseCheckpoint } from './checkpoint';
import { createResponseCompletion } from './completion';
import { createResponseRetirement } from './retirement';
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
const owner = { userId: new mongoose.Types.ObjectId().toString(), tenantId: 'completion-tenant' };
const scoped = <T>(work: () => Promise<T>) => tenantStorage.run(owner, work);
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 1 },
instanceOpts: [{ args: ['--nounixsocket'] }],
});
await mongoose.connect(mongo.getUri(), { autoIndex: false });
createModels(mongoose);
storage = createResponseStorage(mongoose);
});
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.ResponseScope?.deleteMany({});
await mongoose.models.User.deleteMany({});
});
await scoped(async () => {
await mongoose.models.User.create({
_id: owner.userId,
tenantId: owner.tenantId,
email: 'completion-owner@example.com',
provider: 'local',
});
});
});
afterEach(() => {
jest.restoreAllMocks();
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
async function setup(): Promise<{
record: StoredResponseRecord;
receipt: ResponseTerminalReceipt;
}> {
const reserved = await storage.reserve({
...owner,
responseId: 'resp_completion',
requestKey: 'original-request',
fingerprint: 'original',
agentId: 'agent',
sourceKeyId: 'original-key',
conversationId: 'conversation',
parentMessageId: 'root',
requestJSON: JSON.stringify({
model: 'agent',
input: 'Original input',
background: true,
store: true,
}),
});
if (!reserved.ok) throw new Error('Reservation failed');
const initial = {
...buildPendingBackgroundResponse(reserved.record),
status: 'in_progress' as const,
};
const running = await storage.publish(owner, reserved.record.responseId, 0, {
status: 'in_progress',
generationCreatedAt: 10,
responseJSON: JSON.stringify(initial),
});
if (!running.ok) throw new Error('Publication failed');
const response = {
...initial,
status: 'completed' as const,
output: [
{
type: 'message',
id: 'message',
role: 'assistant',
status: 'completed',
content: [{ type: 'output_text', text: 'saved final answer', annotations: [] }],
},
],
};
return {
record: running.record,
receipt: {
generationCreatedAt: 10,
providerExecutionId: 'provider',
ownerId: 'winning-worker',
status: 'completed',
responseJSON: JSON.stringify(response),
checkpointNamespace: 'owned-scope',
},
};
}
it('retains the exact completed answer across cleanup failure and a fresh recovery instance', () =>
scoped(async () => {
const { record, receipt } = await setup();
const cleanup = jest.fn(async () => {});
cleanup.mockRejectedValueOnce(new Error('checkpoint unavailable'));
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: cleanup,
});
await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow(
'checkpoint unavailable',
);
const pending = (await storage.read(owner, record.responseId))!;
expect(pending).toMatchObject({ status: 'in_progress', pendingTerminal: receipt });
expect(pending.responseJSON).not.toContain('saved final answer');
const restarted = createResponseCompletion({
releaseExecution: async () => {},
storage: createResponseStorage(mongoose),
cleanupCheckpoint: cleanup,
});
const complete = await restarted.finish(pending);
expect(complete).toMatchObject({
status: 'completed',
responseJSON: receipt.responseJSON,
requestKey: 'original-request',
});
expect(complete.pendingTerminal).toBeUndefined();
expect(cleanup).toHaveBeenCalledTimes(2);
}));
it('recovers initial-execution checkpoint cleanup without a review record or current deployment settings', () =>
scoped(async () => {
const { record, receipt } = await setup();
const namespace = createCheckpointNamespace(owner.userId, owner.tenantId);
const sibling = createCheckpointNamespace(owner.userId, owner.tenantId);
const settings = {
type: 'mongo',
checkpointCollectionName: 'initial_receipt_checkpoints',
checkpointWritesCollectionName: 'initial_receipt_writes',
};
const savedReceipt = {
...receipt,
checkpointNamespace: namespace,
checkpointConfigJSON: JSON.stringify(settings),
};
const collections = [
settings.checkpointCollectionName,
settings.checkpointWritesCollectionName,
];
for (const name of collections) {
const collection = mongoose.connection.db!.collection(name);
await collection.deleteMany({});
await collection.insertMany(
[namespace, sibling].map((scope) => ({
thread_id: record.conversationId,
checkpoint_ns: scope,
checkpoint_id: scope,
})),
);
}
const unavailable = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {
throw new Error('checkpoint storage temporarily unavailable');
},
});
await expect(unavailable.stageAndFinish(record, savedReceipt)).rejects.toThrow(
'temporarily unavailable',
);
const pending = (await storage.read(owner, record.responseId))!;
expect(pending.review).toBeUndefined();
expect(pending.pendingTerminal).toEqual(savedReceipt);
const restarted = createResponseCompletion({
releaseExecution: async () => {},
storage: createResponseStorage(mongoose),
cleanupCheckpoint: cleanupResponseCheckpoint,
});
expect((await restarted.finish(pending)).status).toBe('completed');
for (const name of collections) {
const remaining = await mongoose.connection.db!.collection(name).find({}).toArray();
expect(remaining).toHaveLength(1);
expect(remaining[0].checkpoint_ns).toBe(sibling);
}
}));
it('retries terminal publication without losing the answer after cleanup already succeeded', () =>
scoped(async () => {
const { record, receipt } = await setup();
const cleanup = jest.fn(async () => {});
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: cleanup,
});
jest
.spyOn(storage, 'finishTerminal')
.mockRejectedValueOnce(new Error('publication unavailable'));
await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow(
'publication unavailable',
);
const pending = (await storage.read(owner, record.responseId))!;
expect(pending.pendingTerminal?.responseJSON).toBe(receipt.responseJSON);
expect((await completion.finish(pending)).responseJSON).toBe(receipt.responseJSON);
expect(cleanup).toHaveBeenCalledTimes(2);
}));
it('reconciles a lost finish acknowledgement and retries execution release on a later terminal read', () =>
scoped(async () => {
const { record, receipt } = await setup();
const finish = storage.finishTerminal.bind(storage);
jest.spyOn(storage, 'finishTerminal').mockImplementationOnce(async (...args) => {
await finish(...args);
throw new Error('finish acknowledgement lost');
});
const release = jest.fn(async () => {});
release.mockRejectedValueOnce(new Error('release transport unavailable'));
const completion = createResponseCompletion({
storage,
cleanupCheckpoint: async () => {},
releaseExecution: release,
});
await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow(
'release transport unavailable',
);
const published = (await storage.read(owner, record.responseId))!;
expect(published.status).toBe('completed');
expect(published.pendingTerminal).toBeUndefined();
expect((await completion.finish(published)).responseJSON).toBe(receipt.responseJSON);
expect(release).toHaveBeenCalledTimes(2);
}));
it('reconciles a committed terminal stage whose acknowledgement was lost', () =>
scoped(async () => {
const { record, receipt } = await setup();
const stage = storage.stageTerminal.bind(storage);
const spy = jest.spyOn(storage, 'stageTerminal').mockImplementationOnce(async (...args) => {
await stage(...args);
throw new Error('lost stage acknowledgement');
});
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
expect(await completion.stageAndFinish(record, receipt)).toMatchObject({
status: 'completed',
responseJSON: receipt.responseJSON,
});
expect(spy).toHaveBeenCalledTimes(1);
}));
it.each([false, true])(
'stages by execution identity despite projection changes after provider drain (late Stop: %s)',
(lateStop) =>
scoped(async () => {
const { record, receipt } = await setup();
const projected = await storage.publish(owner, record.responseId, record.revision, {
status: 'in_progress',
responseJSON: JSON.stringify({
...JSON.parse(record.responseJSON!),
recovery: 'execution_state_unavailable',
}),
});
expect(projected.ok).toBe(true);
if (lateStop) await storage.requestCancellation(owner, record.responseId, Date.now());
const stage = jest.spyOn(storage, 'stageTerminal');
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
const result = await completion.stageAndFinish(record, receipt);
expect(result).toMatchObject({ status: 'completed', responseJSON: receipt.responseJSON });
expect(stage).toHaveBeenCalledTimes(1);
}),
);
it('does not replace another terminal receipt after a staging conflict', () =>
scoped(async () => {
const { record, receipt } = await setup();
const competing = { ...receipt, ownerId: 'another-worker' };
expect(
(await storage.stageTerminal(owner, record.responseId, record.revision, competing)).ok,
).toBe(true);
const stage = jest.spyOn(storage, 'stageTerminal');
const cleanup = jest.fn(async () => {});
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: cleanup,
});
await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow(
'Response terminal receipt publication conflicted',
);
expect(stage).toHaveBeenCalledTimes(1);
expect(cleanup).not.toHaveBeenCalled();
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(competing);
}));
it('does not retry a receipt from a different generation', () =>
scoped(async () => {
const { record, receipt } = await setup();
const stage = jest.spyOn(storage, 'stageTerminal');
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
await expect(
completion.stageAndFinish(
{ ...record, generationCreatedAt: 9 },
{ ...receipt, generationCreatedAt: 9 },
),
).rejects.toThrow('Response terminal receipt publication conflicted');
expect(stage).toHaveBeenCalledTimes(1);
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined();
}));
it.each(['provider', 'another-provider'])(
'does not retry across a new review checkpoint owned by %s',
(providerExecutionId) =>
scoped(async () => {
const { record, receipt } = await setup();
const reviewed = await storage.publishReview(
owner,
record.responseId,
record.revision,
receipt.generationCreatedAt,
{
actionId: 'new-action',
providerExecutionId,
expiresAt: Date.now() + 60_000,
partialResponseJSON: record.responseJSON!,
},
record.responseJSON!,
);
expect(reviewed.ok).toBe(true);
const stage = jest.spyOn(storage, 'stageTerminal');
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow(
'Response terminal receipt publication conflicted',
);
expect(stage).toHaveBeenCalledTimes(1);
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined();
}),
);
it.each([false, true])(
'does not retry a terminal receipt after a new resume intent (admitted: %s)',
(admitted) =>
scoped(async () => {
const { record, receipt } = await setup();
const reviewed = await storage.publishReview(
owner,
record.responseId,
record.revision,
receipt.generationCreatedAt,
{
actionId: 'action',
providerExecutionId: receipt.providerExecutionId,
expiresAt: Date.now() + 60_000,
partialResponseJSON: record.responseJSON!,
},
record.responseJSON!,
);
if (!reviewed.ok) throw new Error('Review publication failed');
const reserved = await storage.reserveResume(
owner,
record.responseId,
{
generationCreatedAt: receipt.generationCreatedAt,
intentId: 'intent',
actionId: 'action',
previousProviderExecutionId: receipt.providerExecutionId,
providerExecutionId: 'next-provider',
fingerprint: 'decision',
resolutionJSON: '{}',
},
Date.now(),
);
expect(reserved.ok).toBe(true);
if (admitted)
expect(
(
await storage.admitResume(
owner,
record.responseId,
'intent',
receipt.generationCreatedAt,
'next-provider',
record.responseJSON!,
Date.now(),
)
).ok,
).toBe(true);
const stage = jest.spyOn(storage, 'stageTerminal');
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
await expect(completion.stageAndFinish(reviewed.record, receipt)).rejects.toThrow(
'Response terminal receipt publication conflicted',
);
expect(stage).toHaveBeenCalledTimes(1);
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined();
}),
);
it('lets authorized polling recover a saved result without queueing or observing execution', () =>
scoped(async () => {
const { record, receipt } = await setup();
await storage.stageTerminal(owner, record.responseId, record.revision, receipt);
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
const enqueue = jest.fn(async () => {
throw new Error('No model redelivery');
});
const observe = jest.fn(async () => {
throw new Error('Do not reconstruct progress');
});
const service = createBackgroundResponseService({
storage,
authorize: async () => true,
enqueue,
observe,
resolveContinuation: async () => ({
conversationId: 'conversation',
parentMessageId: 'root',
}),
cancel: async () => {},
recoverTerminal: completion.finish,
});
const result = await service.retrieve(owner, record.responseId);
expect(result).toMatchObject({ ok: true, response: JSON.parse(receipt.responseJSON) });
expect(enqueue).not.toHaveBeenCalled();
expect(observe).not.toHaveBeenCalled();
}));
it('does not let a late Stop overwrite the saved result or require a transient job to survive', () =>
scoped(async () => {
const { record, receipt } = await setup();
await storage.stageTerminal(owner, record.responseId, record.revision, receipt);
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
const getJob = jest.fn(async () => {
throw new Error('Transient job may have expired');
});
const abortJob = jest.fn(async () => {
throw new Error('No provider needs stopping');
});
const retire = createResponseRetirement({
storage,
jobs: { getJob, abortJob, finishProviderTerminalPersistence: jest.fn() },
recoverTerminal: completion.finish,
});
expect(await retire(owner, record.responseId)).toBe('stopped');
expect(await storage.read(owner, record.responseId)).toMatchObject({
status: 'completed',
responseJSON: receipt.responseJSON,
});
expect(getJob).not.toHaveBeenCalled();
expect(abortJob).not.toHaveBeenCalled();
}));
it('does not recover or disclose a saved reply after source authorization is revoked', () =>
scoped(async () => {
const { record, receipt } = await setup();
await storage.stageTerminal(owner, record.responseId, record.revision, receipt);
const recoverTerminal = jest.fn(async () => {
throw new Error('Unauthorized recovery');
});
const cancel = jest.fn(async () => {});
const service = createBackgroundResponseService({
storage,
authorize: async () => false,
enqueue: async () => {},
observe: async () => null,
resolveContinuation: async () => ({
conversationId: 'conversation',
parentMessageId: 'root',
}),
cancel,
recoverTerminal,
});
const denied = { ok: false, status: 404, code: 'response_not_found' };
expect(await service.retrieve(owner, record.responseId)).toEqual(denied);
expect(await service.cancel(owner, record.responseId)).toEqual(denied);
expect(recoverTerminal).not.toHaveBeenCalled();
expect(cancel).not.toHaveBeenCalled();
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(receipt);
}));
it('converges concurrent finishers on one immutable canonical result', () =>
scoped(async () => {
const { record, receipt } = await setup();
const staged = await storage.stageTerminal(owner, record.responseId, record.revision, receipt);
if (!staged.ok) throw new Error('Stage failed');
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
const results = await Promise.all(
Array.from({ length: 12 }, () => completion.finish(staged.record)),
);
expect(
results.every(
(result) => result.status === 'completed' && result.responseJSON === receipt.responseJSON,
),
).toBe(true);
expect((await storage.read(owner, record.responseId))?.revision).toBe(
staged.record.revision + 1,
);
}));
it('rejects invalid receipt identity before it can poison the durable journal', () =>
scoped(async () => {
const { record, receipt } = await setup();
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
await expect(
completion.stageAndFinish(record, { ...receipt, responseJSON: 'not JSON' }),
).rejects.toThrow('Response terminal receipt is invalid');
await expect(
completion.stageAndFinish(record, { ...receipt, generationCreatedAt: 11 }),
).rejects.toThrow('Response terminal receipt identity changed');
expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined();
}));
it.each(['owner', 'conversation'] as const)(
'discards a drained terminal payload after durable %s retirement without expiry',
(kind) =>
scoped(async () => {
const { record, receipt } = await setup();
const db = createMethods(mongoose);
if (kind === 'owner') await db.deleteUserById(owner.userId);
else await db.deleteConvos(owner.userId, { conversationId: record.conversationId });
expect(await storage.read(owner, record.responseId)).toBeNull();
const cleanup = jest.fn(async () => {});
const release = jest.fn(async () => {});
const completion = createResponseCompletion({
storage,
cleanupCheckpoint: cleanup,
releaseExecution: release,
});
const retired = await completion.stageAndFinish(record, receipt);
expect(retired).toMatchObject({
status: 'completed',
requestJSON: '',
retirement: kind === 'owner' ? 'owner_deleted' : 'conversation_deleted',
});
expect(retired.responseJSON).toBeUndefined();
expect(retired.pendingTerminal).toBeUndefined();
expect(cleanup).toHaveBeenCalledTimes(1);
expect(release).toHaveBeenCalledWith(
expect.objectContaining({
retirement: kind === 'owner' ? 'owner_deleted' : 'conversation_deleted',
}),
receipt,
);
expect(await storage.read(owner, record.responseId)).toBeNull();
}),
);
it.each(['owner', 'conversation'] as const)(
'returns not found when an authorized read races %s retirement during recovery',
(kind) =>
scoped(async () => {
const { record, receipt } = await setup();
await storage.stageTerminal(owner, record.responseId, record.revision, receipt);
const completion = createResponseCompletion({
releaseExecution: async () => {},
storage,
cleanupCheckpoint: async () => {},
});
const observe = jest.fn();
const enqueue = jest.fn();
const service = createBackgroundResponseService({
storage,
authorize: async () => true,
observe,
enqueue,
resolveContinuation: async () => ({
conversationId: record.conversationId,
parentMessageId: 'root',
}),
cancel: async () => {},
recoverTerminal: async (retained) => {
const db = createMethods(mongoose);
if (kind === 'owner') await db.deleteUserById(owner.userId);
else await db.deleteConvos(owner.userId, { conversationId: record.conversationId });
return completion.finish(retained);
},
});
const result = await service.retrieve(owner, record.responseId);
expect(result).toEqual({ ok: false, status: 404, code: 'response_not_found' });
expect(JSON.stringify(result)).not.toContain('saved final answer');
expect(observe).not.toHaveBeenCalled();
expect(enqueue).not.toHaveBeenCalled();
}),
);

View file

@ -0,0 +1,143 @@
import { tenantStorage } from '@librechat/data-schemas';
import type { StoredResponseRecord, ResponseTerminalReceipt } from '@librechat/data-schemas';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { createResponseExecutionRelease } from './completion';
const owner = { userId: 'release-owner', tenantId: 'release-tenant' };
const scope = <T>(work: () => Promise<T>) => tenantStorage.run(owner, work);
let jobs: GenerationJobManagerClass;
beforeEach(() => {
jobs = new GenerationJobManagerClass();
jobs.configure({
jobStore: new InMemoryJobStore(),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
jobs.initialize();
});
afterEach(async () => {
await jobs.destroy();
jest.restoreAllMocks();
});
async function fixture() {
const responseId = 'resp_release';
const job = await jobs.createJob(responseId, owner.userId, 'conversation', {
initialMetadata: { endpoint: 'responses', agent_id: 'agent', responseMessageId: responseId },
});
const providerExecutionId = job.metadata.providerExecutionId!;
await jobs.beginProviderExecution(responseId, job.createdAt, providerExecutionId);
await jobs.completeJob(responseId, undefined, job.createdAt, {
expectedProviderExecutionId: providerExecutionId,
persistencePending: true,
});
await jobs.markProviderExecutionDrained(responseId, job.createdAt, providerExecutionId, true);
const record: StoredResponseRecord = {
...owner,
responseId,
agentId: 'agent',
sourceKeyId: 'source',
requestKey: 'request',
requestJSON: '{}',
fingerprint: 'fingerprint',
conversationId: 'conversation',
parentMessageId: 'parent',
status: 'completed',
generationCreatedAt: job.createdAt,
revision: 1,
createdAt: new Date(),
updatedAt: new Date(),
responseJSON: '{}',
};
const receipt: ResponseTerminalReceipt = {
generationCreatedAt: job.createdAt,
providerExecutionId,
status: 'completed',
responseJSON: '{}',
};
expect((await jobs.getJob(responseId))?.metadata.terminalPersistencePending).toBe(true);
return { record, receipt, release: createResponseExecutionRelease(jobs) };
}
it('releases exact post-publication terminal proof with production default cleanup enabled', () =>
scope(async () => {
const f = await fixture();
await f.release(f.record, f.receipt);
expect(await jobs.getJob(f.record.responseId)).toBeUndefined();
await expect(f.release(f.record, f.receipt)).resolves.toBeUndefined();
}));
it('recovers cleanup when Mongo already published and removed its pending receipt', () =>
scope(async () => {
const f = await fixture();
expect(f.record.pendingTerminal).toBeUndefined();
await f.release(f.record);
expect(await jobs.getJob(f.record.responseId)).toBeUndefined();
}));
it.each([
{ userId: 'another-owner' },
{ tenantId: 'another-tenant' },
{ agentId: 'another-agent' },
{ conversationId: 'another-conversation' },
{ generationCreatedAt: 1 },
])('does not release another execution for mismatched record %j', (change) =>
scope(async () => {
const f = await fixture();
await expect(f.release({ ...f.record, ...change }, f.receipt)).rejects.toThrow(
'identity changed',
);
expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).toBe(
true,
);
}),
);
it('does not release proof for another provider attempt', () =>
scope(async () => {
const f = await fixture();
await expect(
f.release(f.record, { ...f.receipt, providerExecutionId: 'another-provider' }),
).rejects.toThrow('identity changed');
expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).toBe(
true,
);
}));
it('does not treat a rejected release acknowledgement as success', () =>
scope(async () => {
const f = await fixture();
jest.spyOn(jobs, 'finishProviderTerminalPersistence').mockResolvedValueOnce(false);
await expect(f.release(f.record, f.receipt)).rejects.toThrow('release is unavailable');
expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).toBe(
true,
);
await f.release(f.record, f.receipt);
expect(await jobs.getJob(f.record.responseId)).toBeUndefined();
}));
it('retries job cleanup after the marker was cleared but deletion failed', () =>
scope(async () => {
const f = await fixture();
jest
.spyOn(jobs.getJobStore(), 'deleteJob')
.mockRejectedValueOnce(new Error('transient deletion failure'));
await expect(f.release(f.record, f.receipt)).rejects.toThrow('transient deletion failure');
expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).not.toBe(
true,
);
await f.release(f.record);
expect(await jobs.getJob(f.record.responseId)).toBeUndefined();
}));
it.each(['owner_deleted', 'conversation_deleted', 'expired'] as const)(
'releases only exact drained proof for a durable %s cleanup identity',
(retirement) =>
scope(async () => {
const f = await fixture();
await f.release({ ...f.record, status: 'in_progress', retirement }, f.receipt);
expect(await jobs.getJob(f.record.responseId)).toBeUndefined();
}),
);

View file

@ -0,0 +1,374 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createModels,
createMethods,
createResponseStorage,
tenantStorage,
runAsSystem,
} from '@librechat/data-schemas';
import type { ResponseStorage, ResponseOwner } from '@librechat/data-schemas';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { createResponseCompletion, createResponseExecutionRelease } from './completion';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { buildPendingBackgroundResponse } from './background';
import { enrollAgentExecution } from '../remote/lifecycle';
import { createResponseRetirement } from './retirement';
const owner = { userId: new mongoose.Types.ObjectId().toString(), tenantId: 'stop-tenant' };
const scoped = <T>(work: () => Promise<T>) => tenantStorage.run(owner, work);
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
let jobs: GenerationJobManagerClass;
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 1 },
instanceOpts: [{ args: ['--nounixsocket'] }],
});
await mongoose.connect(mongo.getUri(), { autoIndex: false });
createModels(mongoose);
storage = createResponseStorage(mongoose);
});
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.ResponseScope?.deleteMany({});
});
await scoped(async () => {
await mongoose.models.User.deleteMany({});
await mongoose.models.User.create({
_id: owner.userId,
tenantId: owner.tenantId,
email: 'stop-owner@example.com',
provider: 'local',
});
});
jobs = new GenerationJobManagerClass();
jobs.configure({
jobStore: new InMemoryJobStore(),
eventTransport: new InMemoryEventTransport(),
});
jobs.initialize();
});
afterEach(async () => {
jest.restoreAllMocks();
await jobs.destroy();
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
async function fixture(principal: ResponseOwner = owner, retainTerminalPersistence = true) {
const responseId = 'resp_running_stop';
const execution = await enrollAgentExecution(
{
runId: responseId,
userId: principal.userId,
conversationId: 'conversation',
agentId: 'agent',
protocol: 'responses',
...(retainTerminalPersistence ? { retainTerminalPersistence: true as const } : {}),
isPrincipalActive: async () => true,
},
{ manager: jobs },
);
const reserved = await storage.reserve({
...principal,
responseId,
requestKey: 'original-request',
fingerprint: 'original',
agentId: 'agent',
sourceKeyId: 'original-key',
conversationId: 'conversation',
parentMessageId: 'root',
requestJSON: JSON.stringify({ model: 'agent', input: 'Test', background: true, store: true }),
});
if (!reserved.ok) throw new Error('Reservation failed');
const running = await storage.publish(principal, responseId, 0, {
status: 'in_progress',
generationCreatedAt: execution.createdAt,
responseJSON: JSON.stringify({
...buildPendingBackgroundResponse(reserved.record),
status: 'in_progress',
}),
});
if (!running.ok) throw new Error('Publication failed');
await execution.beginProviderExecution();
const completion = createResponseCompletion({
storage,
cleanupCheckpoint: async () => {},
releaseExecution: createResponseExecutionRelease(jobs),
});
const retire = createResponseRetirement({
storage,
jobs,
recoverTerminal: completion.finish,
});
/** Force the live failure ordering: provider settles before abort's chat publication. */
const abort = jobs.abortJob.bind(jobs);
jest.spyOn(jobs, 'abortJob').mockImplementationOnce((id, options) =>
abort(id, {
...options,
beforePublish: async (result) => {
expect(execution.signal.aborted).toBe(true);
await execution.settle();
expect((await jobs.getJob(id))?.metadata.providerDrained).toBe(true);
await options?.beforePublish?.(result);
},
}),
);
return { execution, responseId, retire, completion };
}
it('retains a running Stop after provider drain until canonical cancellation releases it with default cleanup', () =>
scoped(async () => {
const f = await fixture();
const publishChat = jest.spyOn(jobs, 'publishTerminalClaim');
const stage = storage.stageTerminal.bind(storage);
jest.spyOn(storage, 'stageTerminal').mockImplementationOnce(async (...args) => {
expect(await jobs.getJob(f.responseId)).toMatchObject({
status: 'aborted',
createdAt: f.execution.createdAt,
metadata: { providerDrained: true, terminalPersistencePending: true },
});
expect(await storage.read(owner, f.responseId)).toMatchObject({
status: 'in_progress',
cancellationRequestedAt: expect.any(Number),
});
return stage(...args);
});
expect(await f.retire(owner, f.responseId)).toBe('stopped');
expect(publishChat).not.toHaveBeenCalled();
expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled');
expect(await jobs.getJob(f.responseId)).toBeUndefined();
expect(await f.retire(owner, f.responseId)).toBe('stopped');
}));
it('drains expired work through metadata identity without publishing private cancellation output', () =>
scoped(async () => {
const f = await fixture();
await mongoose.models.StoredResponse.updateOne(
{ responseId: f.responseId },
{
$set: { expiredAt: new Date(Date.now() - 1) },
},
);
expect(await storage.read(owner, f.responseId)).toBeNull();
const stage = jest.spyOn(storage, 'stageTerminal');
const release = jest.spyOn(jobs, 'finishProviderTerminalPersistence');
expect(await f.retire(owner, f.responseId)).toBe('stopped');
expect(stage).not.toHaveBeenCalled();
expect(release).toHaveBeenCalledTimes(1);
expect(f.execution.signal.aborted).toBe(true);
expect(await jobs.getJob(f.responseId)).toBeUndefined();
expect(await storage.read(owner, f.responseId)).toBeNull();
}));
it('keeps drained Stop proof through canonical receipt failure and retry without executing again', () =>
scoped(async () => {
const f = await fixture();
jest.spyOn(storage, 'stageTerminal').mockRejectedValue(new Error('receipt unavailable'));
await expect(f.retire(owner, f.responseId)).rejects.toThrow('receipt unavailable');
expect(await jobs.getJob(f.responseId)).toMatchObject({
status: 'aborted',
metadata: { providerDrained: true, terminalPersistencePending: true },
});
jest.mocked(storage.stageTerminal).mockRestore();
expect(await f.retire(owner, f.responseId)).toBe('stopped');
expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled');
expect(await jobs.getJob(f.responseId)).toBeUndefined();
}));
it('drains through the retained journal after conversation erasure closes admission', () =>
scoped(async () => {
const f = await fixture();
createModels(mongoose);
await mongoose.models.Conversation.create({
conversationId: 'conversation',
user: owner.userId,
tenantId: owner.tenantId,
endpoint: 'agents',
});
const db = createMethods(mongoose);
await db.deleteConvos(
owner.userId,
{ conversationId: 'conversation' },
{
beforeDelete: async () => {
expect((await storage.read(owner, f.responseId))?.cancellationRequestedAt).toEqual(
expect.any(Number),
);
expect(await f.retire(owner, f.responseId)).toBe('stopped');
},
},
);
expect(await storage.read(owner, f.responseId)).toBeNull();
expect(await jobs.getJob(f.responseId)).toBeUndefined();
}));
it('does not relabel an already-completed provider as cancelled before its canonical publication', () =>
scoped(async () => {
const f = await fixture();
await f.execution.settle();
const stage = jest.spyOn(storage, 'stageTerminal');
expect(await f.retire(owner, f.responseId)).toBe('pending');
expect(stage).not.toHaveBeenCalled();
expect((await jobs.getJob(f.responseId))?.status).toBe('complete');
expect((await storage.read(owner, f.responseId))?.status).toBe('in_progress');
}));
it('repeated Stop recovers a running runtime receipt failure without a paused review or a new execution', () =>
scoped(async () => {
const f = await fixture();
await storage.requestCancellation(owner, f.responseId, Date.now());
await jobs.abortJob(f.responseId, {
expectedCreatedAt: f.execution.createdAt,
awaitProviderDrain: true,
retainTerminalPersistence: true,
});
const identity = await f.execution.confirmTerminalIdentity();
expect(identity.outcome).toBe('aborted');
const record = (await storage.read(owner, f.responseId))!;
expect(record.review).toBeUndefined();
jest.spyOn(storage, 'stageTerminal').mockRejectedValueOnce(new Error('runtime receipt failed'));
await expect(
f.completion.stageAndFinish(record, {
...identity,
status: 'cancelled',
responseJSON: JSON.stringify({
...buildPendingBackgroundResponse(record),
status: 'cancelled',
}),
}),
).rejects.toThrow('runtime receipt failed');
expect((await storage.read(owner, f.responseId))?.pendingTerminal).toBeUndefined();
expect((await jobs.getJob(f.responseId))?.metadata.terminalPersistencePending).toBe(true);
expect(await f.retire(owner, f.responseId)).toBe('stopped');
expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled');
expect(await jobs.getJob(f.responseId)).toBeUndefined();
}));
it('preserves ordinary abort publication and default cleanup when retention is not requested', () =>
scoped(async () => {
const f = await fixture(owner, false);
const publishChat = jest.spyOn(jobs, 'publishTerminalClaim');
expect(await jobs.abortJob(f.responseId, { awaitProviderDrain: true })).toMatchObject({
success: true,
finalEvent: { aborted: true },
});
expect(publishChat).toHaveBeenCalledTimes(1);
expect(await jobs.getJob(f.responseId)).toBeUndefined();
await expect(f.execution.confirmTerminalIdentity()).rejects.toThrow(
'terminal settlement could not be confirmed',
);
}));
it('rejects retained abort on unsupported stores before changing the owned generation', () =>
scoped(async () => {
const f = await fixture();
Object.defineProperty(jobs.getJobStore(), 'providerTerminalPersistence', {
value: false,
configurable: true,
});
await expect(f.retire(owner, f.responseId)).rejects.toThrow('barrier is unavailable');
expect((await jobs.getJob(f.responseId))?.status).toBe('running');
expect(f.execution.signal.aborted).toBe(false);
Object.defineProperty(jobs.getJobStore(), 'providerTerminalPersistence', { value: true });
await f.execution.settle();
}));
it('releases retained provider proof through the complete fallback recovery', () =>
scoped(async () => {
const f = await fixture();
const retire = createResponseRetirement({ storage, jobs, cleanupCheckpoint: async () => {} });
expect(await retire(owner, f.responseId)).toBe('stopped');
expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled');
expect(await jobs.getJob(f.responseId)).toBeUndefined();
expect(await retire(owner, f.responseId)).toBe('stopped');
}));
it('keeps the fallback retention barrier until failed checkpoint cleanup is retried', () =>
scoped(async () => {
const f = await fixture();
await f.execution.settle();
const record = (await storage.read(owner, f.responseId))!;
const job = (await jobs.getJob(f.responseId))!;
const receipt = {
generationCreatedAt: f.execution.createdAt,
providerExecutionId: job.metadata.providerExecutionId!,
ownerId: job.metadata.providerExecutionOwnerId,
status: 'completed' as const,
responseJSON: JSON.stringify({
...buildPendingBackgroundResponse(record),
status: 'completed',
}),
checkpointNamespace: 'owned-scope',
checkpointConfigJSON: '{"pinned":true}',
};
expect(
(await storage.stageTerminal(owner, f.responseId, { execution: record }, receipt)).ok,
).toBe(true);
const cleanup = jest.fn(async () => {});
cleanup.mockRejectedValueOnce(new Error('checkpoint unavailable'));
const retire = createResponseRetirement({ storage, jobs, cleanupCheckpoint: cleanup });
await expect(retire(owner, f.responseId)).rejects.toThrow('checkpoint unavailable');
expect((await jobs.getJob(f.responseId))?.metadata.terminalPersistencePending).toBe(true);
expect((await storage.read(owner, f.responseId))?.pendingTerminal).toEqual(receipt);
expect(await retire(owner, f.responseId)).toBe('stopped');
expect(cleanup).toHaveBeenCalledTimes(2);
expect(await jobs.getJob(f.responseId)).toBeUndefined();
}));
it.each(['owner', 'conversation'] as const)(
'drains a retained provider after %s erasure without an expiry deadline',
(kind) =>
scoped(async () => {
const f = await fixture();
const db = createMethods(mongoose);
if (kind === 'owner') await db.deleteUserById(owner.userId);
else await db.deleteConvos(owner.userId, { conversationId: 'conversation' });
expect(await storage.read(owner, f.responseId)).toBeNull();
const job = (await jobs.getJob(f.responseId))!;
expect(
await storage.readCleanupIdentity(owner, f.responseId, {
...owner,
responseId: f.responseId,
agentId: 'agent',
conversationId: 'conversation',
generationCreatedAt: job.createdAt,
}),
).toMatchObject({ retirement: kind === 'owner' ? 'owner_deleted' : 'conversation_deleted' });
expect(await f.retire(owner, f.responseId)).toBe('stopped');
expect(await jobs.getJob(f.responseId)).toBeUndefined();
expect(await storage.read(owner, f.responseId)).toBeNull();
}),
);
it.each([null, ''])(
'retires and releases real provider state for legacy persisted tenant %p',
async (tenantId) => {
const principal = { userId: new mongoose.Types.ObjectId().toString() };
await tenantStorage.run(principal, async () => {
await mongoose.models.User.create({
_id: principal.userId,
email: 'legacy-stop@example.com',
provider: 'local',
});
const f = await fixture(principal);
await mongoose.models.StoredResponse.collection.updateOne(
{ responseId: f.responseId, userId: principal.userId },
{ $set: { tenantId } },
);
const record = await storage.read(principal, f.responseId);
expect(record?.tenantId).toBe(tenantId);
expect((await jobs.getJob(f.responseId))?.metadata.tenantId).toBeUndefined();
expect(await f.retire(principal, f.responseId)).toBe('stopped');
expect((await storage.read(principal, f.responseId))?.status).toBe('cancelled');
expect(await jobs.getJob(f.responseId)).toBeUndefined();
expect(await f.retire(principal, f.responseId)).toBe('stopped');
});
},
);

View file

@ -0,0 +1,256 @@
import type { ResponseOwner, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import type { ResponseCheckpointIdentity } from './checkpoint';
import type { ResponseCompletionRecord } from './completion';
import { createResponseCompletion, createResponseExecutionRelease } from './completion';
import { buildPendingBackgroundResponse } from './background';
import { responseTenantId } from './scope';
export interface ResponseRetirementDependencies {
storage: ResponseStorage;
jobs: Pick<
GenerationJobManagerClass,
'getJob' | 'abortJob' | 'finishProviderTerminalPersistence'
>;
/** Required before a durably paused response may publish cancellation. Must be idempotent. */
cleanupCheckpoint?: (
record: ResponseCheckpointIdentity,
checkpointNamespace: string,
) => Promise<void>;
recoverTerminal?: (record: StoredResponseRecord) => Promise<ResponseCompletionRecord>;
now?: () => number;
}
const terminal = (record: StoredResponseRecord) =>
['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status);
/** Internal cleanup authority: may stop one owned response, never execute or disclose its output. */
export function createResponseRetirement(
deps: ResponseRetirementDependencies,
): (owner: ResponseOwner, responseId: string) => Promise<'stopped' | 'pending'> {
const recover =
deps.recoverTerminal ??
createResponseCompletion({
storage: deps.storage,
cleanupCheckpoint:
deps.cleanupCheckpoint ??
(async () => {
throw new Error('Response checkpoint cleanup unavailable');
}),
releaseExecution: createResponseExecutionRelease(deps.jobs),
}).finish;
return async (owner, responseId) => {
let record = await deps.storage.requestCancellation(
owner,
responseId,
(deps.now ?? Date.now)(),
);
if (!record) {
const job = await deps.jobs.getJob(responseId);
const retained =
job &&
job.metadata.userId === owner.userId &&
responseTenantId(job.metadata.tenantId) === responseTenantId(owner.tenantId) &&
job.metadata.endpoint === 'responses' &&
job.metadata.responseMessageId === responseId &&
job.metadata.agent_id &&
job.metadata.conversationId
? {
userId: job.metadata.userId,
tenantId: job.metadata.tenantId,
responseId,
agentId: job.metadata.agent_id,
conversationId: job.metadata.conversationId,
generationCreatedAt: job.createdAt,
checkpointNamespace: job.metadata.checkpointNamespace,
checkpointConfigJSON: job.metadata.responseCheckpointConfigJSON,
}
: undefined;
const identity = await deps.storage.readCleanupIdentity(owner, responseId, retained);
if (
!identity?.retirement &&
!(identity?.expiredAt && identity.expiredAt.getTime() <= (deps.now ?? Date.now)())
)
throw new Error('Response retirement identity unavailable');
if (!job) return identity.generationCreatedAt == null ? 'stopped' : 'pending';
if (
job.createdAt !== identity.generationCreatedAt ||
job.metadata.userId !== identity.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(identity.tenantId) ||
job.metadata.agent_id !== identity.agentId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.conversationId !== identity.conversationId ||
job.metadata.responseMessageId !== responseId
)
throw new Error('Response retirement generation changed');
const providerExecutionId = job.metadata.providerExecutionId;
const providerExecutionOwnerId = job.metadata.providerExecutionOwnerId;
if (!providerExecutionId) throw new Error('Response retirement provider unavailable');
const cleanup = async () => {
if (identity.requiresCheckpointCleanup || identity.checkpointConfigJSON) {
const namespace = identity.checkpointNamespace ?? job.metadata.checkpointNamespace;
if (!namespace || !deps.cleanupCheckpoint || !identity.checkpointConfigJSON)
throw new Error('Response checkpoint cleanup unavailable');
await deps.cleanupCheckpoint(
{
...identity,
pendingTerminal: {
checkpointNamespace: namespace,
checkpointConfigJSON: identity.checkpointConfigJSON,
},
},
namespace,
);
}
};
await deps.jobs.abortJob(responseId, {
expectedCreatedAt: identity.generationCreatedAt,
awaitProviderDrain: true,
retainTerminalPersistence: true,
beforePublish: cleanup,
});
await createResponseExecutionRelease({
getJob: deps.jobs.getJob.bind(deps.jobs),
finishProviderTerminalPersistence: deps.jobs.finishProviderTerminalPersistence.bind(
deps.jobs,
),
})(
{ ...identity, status: 'cancelled' },
{ providerExecutionId, ownerId: providerExecutionOwnerId },
);
return 'stopped';
}
if (terminal(record)) {
await recover(record);
return 'stopped';
}
if (record.pendingTerminal) {
return terminal(await recover(record)) ? 'stopped' : 'pending';
}
if (record.generationCreatedAt == null) {
const response = {
...buildPendingBackgroundResponse(record),
status: 'cancelled' as const,
completed_at: Math.floor((deps.now ?? Date.now)() / 1000),
};
const cancelled = await deps.storage.publish(owner, responseId, record.revision, {
status: 'cancelled',
responseJSON: JSON.stringify(response),
});
// This CAS beats enrollment publication, which must succeed before provider startup.
if (cancelled.ok) return 'stopped';
record = await deps.storage.read(owner, responseId);
if (!record) throw new Error('Response disappeared during retirement');
if (terminal(record)) {
await recover(record);
return 'stopped';
}
}
const job = await deps.jobs.getJob(responseId);
if (!job) return 'pending';
if (
record.generationCreatedAt == null ||
job.createdAt !== record.generationCreatedAt ||
job.metadata.userId !== record.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) ||
job.metadata.agent_id !== record.agentId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.conversationId !== record.conversationId ||
job.metadata.responseMessageId !== record.responseId
)
throw new Error('Response retirement generation changed');
const epoch = record.generationCreatedAt;
const agentId = record.agentId;
const conversationId = record.conversationId;
const providerExecutionId = job.metadata.providerExecutionId;
const providerExecutionOwnerId = job.metadata.providerExecutionOwnerId;
async function stageCancellation(): Promise<void> {
const stopped = await deps.jobs.getJob(responseId);
const proof = stopped?.metadata;
if (
stopped?.status !== 'aborted' ||
!proof ||
proof.providerDrained !== true ||
stopped.createdAt !== epoch ||
proof.userId !== owner.userId ||
responseTenantId(proof.tenantId) !== responseTenantId(owner.tenantId) ||
proof.agent_id !== agentId ||
proof.endpoint !== 'responses' ||
proof.responseMessageId !== responseId ||
proof.conversationId !== conversationId ||
proof.providerExecutionId !== providerExecutionId ||
proof.providerExecutionOwnerId !== providerExecutionOwnerId ||
!proof.providerExecutionId
)
return;
const current = await deps.storage.read(owner, responseId);
if (!current || current.generationCreatedAt !== epoch)
throw new Error('Response retirement journal changed');
if (terminal(current) || current.pendingTerminal) return;
if (current.cancellationRequestedAt == null)
throw new Error('Response cancellation intent is unavailable');
if (current.review && (!proof.checkpointNamespace || !deps.cleanupCheckpoint)) return;
const response = {
...buildPendingBackgroundResponse(current),
model: current.agentId,
status: 'cancelled' as const,
completed_at: Math.floor((deps.now ?? Date.now)() / 1000),
};
const receipt = {
generationCreatedAt: epoch,
providerExecutionId: proof.providerExecutionId,
...(proof.providerExecutionOwnerId == null
? {}
: { ownerId: proof.providerExecutionOwnerId }),
status: 'cancelled' as const,
responseJSON: JSON.stringify(response),
...(current.review
? {
checkpointNamespace: proof.checkpointNamespace,
...(current.review.checkpointConfigJSON == null
? {}
: { checkpointConfigJSON: current.review.checkpointConfigJSON }),
}
: {}),
};
try {
const staged = await deps.storage.stageTerminal(
owner,
responseId,
{ execution: current },
receipt,
);
if (!staged.ok) throw new Error('Response retirement receipt conflicted');
} catch (error) {
const saved = await deps.storage.read(owner, responseId);
if (
!saved ||
saved.generationCreatedAt !== epoch ||
(!terminal(saved) &&
(saved.pendingTerminal?.providerExecutionId !== receipt.providerExecutionId ||
saved.pendingTerminal.responseJSON !== receipt.responseJSON))
)
throw error;
}
}
await deps.jobs.abortJob(responseId, {
expectedCreatedAt: record.generationCreatedAt,
awaitProviderDrain: true,
retainTerminalPersistence: true,
beforePublish: stageCancellation,
});
// Runtime publication includes provider drain and trailing writes; a missing job alone is not proof.
let settled = await deps.storage.read(owner, responseId);
if (settled && terminal(settled)) {
await recover(settled);
return 'stopped';
}
if (settled?.pendingTerminal) return terminal(await recover(settled)) ? 'stopped' : 'pending';
// A repeated Stop can recover a failed runtime receipt only from the exact retained,
// aborted and drained provider. Completed jobs and absent jobs are not cancellation proof.
await stageCancellation();
settled = await deps.storage.read(owner, responseId);
if (settled?.pendingTerminal) settled = await recover(settled);
return settled && terminal(settled) ? 'stopped' : 'pending';
};
}

View file

@ -0,0 +1,14 @@
import { isReservedTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas';
/** Legacy persisted tenantless rows use missing, null or empty values. */
export function responseTenantId(value: string | null | undefined): string | undefined {
return value || undefined;
}
/** Internal partition sentinels never acquire response execution authority. */
export function isResponseTenantId(value: unknown): value is string | null | undefined {
return (
value == null ||
(typeof value === 'string' && value !== SYSTEM_TENANT_ID && !isReservedTenantId(value))
);
}

View file

@ -0,0 +1,172 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createModels,
createResponseStorage,
tenantStorage,
runAsSystem,
} from '@librechat/data-schemas';
import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { BackgroundResponseTransportDependencies } from './transport';
import { createBackgroundResponseTransport } from './transport';
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
let record: StoredResponseRecord;
const owner = { userId: '000000000000000000000001', tenantId: 'transport-test' };
type Fetch = NonNullable<BackgroundResponseTransportDependencies['fetch']>;
const scoped = <T>(work: () => Promise<T>) => tenantStorage.run(owner, work);
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 1 },
instanceOpts: [{ args: ['--nounixsocket'] }],
});
await mongoose.connect(mongo.getUri(), { autoIndex: false });
createModels(mongoose);
storage = createResponseStorage(mongoose);
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.ResponseScope?.deleteMany({});
});
const saved = await scoped(() =>
storage.reserve({
...owner,
responseId: 'resp_test',
agentId: 'agent',
sourceKeyId: 'original-key',
requestKey: 'request',
fingerprint: 'fingerprint',
requestJSON: JSON.stringify({
model: 'agent',
input: 'Private input',
background: true,
store: true,
}),
conversationId: 'conversation',
parentMessageId: 'root',
}),
);
if (!saved.ok) throw new Error('Reservation failed');
record = saved.record;
});
async function enroll() {
const result = await scoped(() =>
storage.publish(owner, record.responseId, record.revision, {
status: 'in_progress',
responseJSON: JSON.stringify(ack),
generationCreatedAt: 1234,
}),
);
if (!result.ok) throw new Error('Enrollment failed');
}
function transport(fetch: Fetch, origin = 'http://127.0.0.1:3088') {
return createBackgroundResponseTransport({
storage,
fetch,
getOrigin: () => origin,
signingSecret: () => 'transport-test-secret',
acknowledgementTimeoutMs: async () => 1000,
});
}
const ack = {
id: 'resp_test',
object: 'response',
model: 'agent',
status: 'in_progress',
background: true,
store: true,
};
it('sends only a server-purpose credential and saved reference through Responses ingress', async () => {
const fetch: Fetch = async (url, init) => {
expect(String(url)).toBe('http://127.0.0.1:3088/api/agents/v1/responses');
expect(init?.body).toBe('{}');
expect(init?.headers).toMatchObject({ 'x-lc-response-worker': '1' });
expect(JSON.stringify(init)).not.toContain('Private input');
await enroll();
return Response.json(ack, { status: 202 });
};
await expect(scoped(() => transport(fetch)(record))).resolves.toEqual({
status: 'started',
generationCreatedAt: 1234,
});
});
it('recovers a lost acknowledgement only when durable enrollment proves execution', async () => {
const fetch: Fetch = async () => {
await enroll();
throw new Error('lost ACK');
};
await expect(scoped(() => transport(fetch)(record))).resolves.toEqual({
status: 'resumed',
generationCreatedAt: 1234,
});
});
it('does not treat a network timeout without enrollment as successful dispatch', async () => {
const fetch: Fetch = async () => {
throw new Error('timeout');
};
await expect(scoped(() => transport(fetch)(record))).rejects.toMatchObject({
code: 'response_acknowledgement_unavailable',
certainty: 'ambiguous',
});
});
it('attaches duplicate admission to the same enrolled generation', async () => {
const fetch: Fetch = async () => {
await enroll();
return new Response('', { status: 409 });
};
await expect(scoped(() => transport(fetch)(record))).resolves.toEqual({
status: 'resumed',
generationCreatedAt: 1234,
});
});
it.each([401, 403, 429, 503])(
'classifies admission HTTP %s without exposing its body',
async (status) => {
const fetch: Fetch = async () => new Response('PRIVATE upstream failure', { status });
await expect(scoped(() => transport(fetch)(record))).rejects.toMatchObject({
message: 'Background response admission rejected',
status,
retryable: status === 429 || status >= 500,
certainty: status >= 500 ? 'ambiguous' : 'definite',
});
},
);
it.each(['not JSON', JSON.stringify({ ...ack, id: 'other-response' }), JSON.stringify(ack)])(
'does not accept an unproved or mismatched acknowledgement',
async (body) => {
const fetch: Fetch = async () => new Response(body, { status: 202 });
await expect(scoped(() => transport(fetch)(record))).rejects.toMatchObject({
code: 'response_acknowledgement_invalid',
});
},
);
it('refuses remote or credential-bearing worker origins before making a request', async () => {
let calls = 0;
const fetch: Fetch = async () => {
calls++;
return Response.json(ack);
};
for (const origin of [
'https://example.com',
'http://user:secret@127.0.0.1:3088',
'file:///tmp/test',
]) {
await expect(scoped(() => transport(fetch, origin)(record))).rejects.toThrow(
'capable local listener',
);
}
expect(calls).toBe(0);
});

View file

@ -0,0 +1,189 @@
import { z } from 'zod';
import type { StoredResponseRecord, ResponseStorage } from '@librechat/data-schemas';
import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import type { BackgroundResponseQueueDependencies } from './queue';
import {
mintBackgroundResponseWorkerToken,
mintResponseResumeWorkerToken,
RESPONSE_WORKER_TOKEN_EXPIRED,
} from './worker';
import { AgentTriggerExecutionError } from '../triggers/host';
import { responseTenantId } from './scope';
export interface BackgroundResponseTransportDependencies {
storage: ResponseStorage;
getOrigin: () => string;
signingSecret: () => string;
acknowledgementTimeoutMs: (record: StoredResponseRecord) => Promise<number>;
fetch?: (
input: Parameters<typeof globalThis.fetch>[0],
init?: Parameters<typeof globalThis.fetch>[1],
) => ReturnType<typeof globalThis.fetch>;
}
const expiredWorkerCredential = z.object({
error: z.object({ code: z.literal(RESPONSE_WORKER_TOKEN_EXPIRED) }),
});
const acknowledgement = z.object({
id: z.string(),
object: z.literal('response'),
model: z.string(),
background: z.literal(true),
store: z.literal(true),
status: z.enum(['queued', 'in_progress', 'completed', 'cancelled', 'failed', 'incomplete']),
});
export function createBackgroundResponseTransport(
deps: BackgroundResponseTransportDependencies,
): BackgroundResponseQueueDependencies['execute'] {
return createResponseWorkerTransport(deps);
}
/** A consumed approval alone is not proof that its selected provider attempt started. */
export function createResponseResumeTransport(
deps: BackgroundResponseTransportDependencies & {
jobs: Pick<GenerationJobManagerClass, 'getJob'>;
},
): BackgroundResponseQueueDependencies['execute'] {
return createResponseWorkerTransport(deps, deps.jobs);
}
/** Only the capable local replica may execute this queue delivery, through Responses ingress. */
function createResponseWorkerTransport(
deps: BackgroundResponseTransportDependencies,
resumeJobs?: Pick<GenerationJobManagerClass, 'getJob'>,
): BackgroundResponseQueueDependencies['execute'] {
async function receipt(record: StoredResponseRecord, current: StoredResponseRecord | null) {
if (current?.generationCreatedAt == null) return;
if (!resumeJobs)
return { status: 'resumed' as const, generationCreatedAt: current.generationCreatedAt };
if (
!record.resume ||
current.generationCreatedAt !== record.generationCreatedAt ||
current.resume?.intentId !== record.resume.intentId ||
current.resume.providerExecutionId !== record.resume.providerExecutionId ||
current.sourceKeyId !== record.sourceKeyId ||
current.agentId !== record.agentId ||
current.conversationId !== record.conversationId
)
return;
const identity = { generationCreatedAt: current.generationCreatedAt };
if (['completed', 'failed', 'incomplete', 'cancelled'].includes(current.status))
return { ...identity, status: 'settled' as const };
if (current.resume.state !== 'admitted') return;
const job = await resumeJobs.getJob(record.responseId);
if (
!job ||
job.createdAt !== current.generationCreatedAt ||
job.metadata.userId !== current.userId ||
responseTenantId(job.metadata.tenantId) !== responseTenantId(current.tenantId) ||
job.metadata.agent_id !== current.agentId ||
job.metadata.conversationId !== current.conversationId ||
job.metadata.responseMessageId !== current.responseId ||
job.metadata.endpoint !== 'responses' ||
job.metadata.providerExecutionId !== current.resume.providerExecutionId ||
job.metadata.providerExecutionStartedId !== current.resume.providerExecutionId ||
!job.metadata.providerExecutionOwnerId
)
return;
return { ...identity, status: 'resumed' as const };
}
return async (
record: StoredResponseRecord,
signal?: AbortSignal,
): Promise<{
status: 'started' | 'resumed' | 'settled';
generationCreatedAt?: number;
}> => {
const origin = new URL(deps.getOrigin());
if (
!['http:', 'https:'].includes(origin.protocol) ||
origin.username ||
origin.password ||
!['127.0.0.1', '::1', '[::1]', 'localhost'].includes(origin.hostname)
)
throw new Error('Background Responses requires the capable local listener');
const timeoutMs = await deps.acknowledgementTimeoutMs(record);
const timeout = AbortSignal.timeout(timeoutMs);
const token = resumeJobs
? mintResponseResumeWorkerToken(record, deps.signingSecret())
: mintBackgroundResponseWorkerToken(record, deps.signingSecret());
let response: globalThis.Response;
try {
response = await (deps.fetch ?? globalThis.fetch)(
new URL('/api/agents/v1/responses', origin),
{
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
'x-lc-response-worker': '1',
},
body: '{}',
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
},
);
} catch {
const current = await deps.storage.read(record, record.responseId);
const confirmed = await receipt(record, current);
if (confirmed) return confirmed;
throw new AgentTriggerExecutionError(
'Background response admission acknowledgement unavailable',
{
mode: 'fire',
certainty: 'ambiguous',
retryable: true,
code: 'response_acknowledgement_unavailable',
},
);
}
const current = await deps.storage.read(record, record.responseId);
const confirmed = await receipt(record, current);
if (response.status === 409 && confirmed) {
await response.body?.cancel();
return confirmed;
}
if (resumeJobs && response.status === 409) {
await response.body?.cancel();
throw new AgentTriggerExecutionError('Response resume start acknowledgement unavailable', {
mode: 'fire',
certainty: 'ambiguous',
retryable: true,
code: 'response_resume_start_unconfirmed',
});
}
if (!response.ok) {
const expired =
response.status === 401 &&
expiredWorkerCredential.safeParse(await response.json().catch(() => null)).success;
// A 401 body has already been consumed; cancelling its locked reader would hide the
// classified authentication outcome with a stream error.
if (response.status !== 401) await response.body?.cancel();
throw new AgentTriggerExecutionError('Background response admission rejected', {
mode: 'fire',
certainty: response.status >= 500 ? 'ambiguous' : 'definite',
retryable: expired || response.status >= 500 || response.status === 429,
code: expired ? RESPONSE_WORKER_TOKEN_EXPIRED : 'response_admission_rejected',
status: response.status,
});
}
const body = acknowledgement.safeParse(await response.json().catch(() => null));
if (
!body.success ||
body.data.id !== record.responseId ||
body.data.model !== record.agentId ||
!confirmed
)
throw new AgentTriggerExecutionError(
'Background response acknowledgement identity unavailable',
{
mode: 'fire',
certainty: 'ambiguous',
retryable: true,
code: 'response_acknowledgement_invalid',
},
);
return resumeJobs ? confirmed : { ...confirmed, status: 'started' };
};
}

View file

@ -0,0 +1,352 @@
import express from 'express';
import jwt from 'jsonwebtoken';
import request from 'supertest';
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createModels,
createMethods,
createResponseStorage,
tenantStorage,
runAsSystem,
getTenantId,
} from '@librechat/data-schemas';
import type { ResponseStorage } from '@librechat/data-schemas';
import type { RequestHandler } from 'express';
import type { ServerRequest } from '~/types/http';
import {
createBackgroundResponseWorkerAuth,
mintBackgroundResponseWorkerToken,
mintResponseResumeWorkerToken,
getBackgroundResponseWorkerRecord,
RESPONSE_WORKER_TOKEN_EXPIRED,
} from './worker';
import { createBackgroundResponseEnvelope, createResponseResumeEnvelope } from './queue';
import { createBackgroundResponseTransport } from './transport';
import { buildPendingBackgroundResponse } from './background';
const secret = 'worker-test-secret';
const userId = new mongoose.Types.ObjectId().toString();
const responseId = 'resp_worker';
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
let methods: ReturnType<typeof createMethods>;
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } });
await mongoose.connect(mongo.getUri(), { autoIndex: false });
createModels(mongoose);
methods = createMethods(mongoose);
storage = createResponseStorage(mongoose);
});
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.ResponseScope.deleteMany({});
await mongoose.models.User.deleteMany({});
});
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
async function fixture(tenantId?: string) {
const owner = { userId, tenantId };
return tenantStorage.run(owner, async () => {
await mongoose.models.User.create({
_id: userId,
tenantId,
email: 'worker@example.com',
provider: 'local',
});
const saved = await storage.reserve({
...owner,
responseId,
agentId: 'agent',
sourceKeyId: 'original-key',
requestKey: 'request',
fingerprint: 'fingerprint',
conversationId: 'conversation',
parentMessageId: 'root',
requestJSON: JSON.stringify({ model: 'agent', input: 'Saved input', background: true }),
});
if (!saved.ok) throw new Error('Fixture reservation failed');
return saved.record;
});
}
function app(contextTenant?: string, onWorker?: RequestHandler) {
const server = express();
const findUser = jest.fn((id: string) => methods.findUser({ _id: id }));
const authorize = jest.fn().mockResolvedValue(true);
const onError = jest.fn();
server.use((req, res, next) => tenantStorage.run({ tenantId: contextTenant }, next));
server.use(
createBackgroundResponseWorkerAuth(
{ storage, signingSecret: () => secret, findUser, authorize, onError },
(req, res) => {
res.status(403).end();
},
),
);
server.post(
'/',
onWorker ??
((req, res) => {
const principal = (req as ServerRequest).user;
if (!principal) throw new Error('Worker principal missing');
res.json({
tenant: getTenantId() ?? null,
userTenant: principal.tenantId ?? null,
userId: principal.id,
body: req.body,
admitted: getBackgroundResponseWorkerRecord(req)?.responseId,
});
}),
);
return { server, findUser, authorize, onError };
}
const legacy = [undefined, null, ''];
it.each(legacy.flatMap((stored) => legacy.map((user) => [stored, user])))(
'admits legacy stored tenant %p and user tenant %p using canonical worker claims',
async (stored, user) => {
await fixture();
await mongoose.models.StoredResponse.collection.updateOne(
{ responseId },
stored === undefined ? { $unset: { tenantId: 1 } } : { $set: { tenantId: stored } },
);
await mongoose.models.User.collection.updateOne(
{ _id: new mongoose.Types.ObjectId(userId) },
user === undefined ? { $unset: { tenantId: 1 } } : { $set: { tenantId: user } },
);
const record = await storage.read({ userId }, responseId);
if (!record) throw new Error('Fixture disappeared');
const token = mintBackgroundResponseWorkerToken(record, secret);
expect(jwt.decode(token)).not.toHaveProperty('tenantId');
const f = app();
const result = await request(f.server)
.post('/')
.set('x-lc-response-worker', '1')
.auth(token, { type: 'bearer' })
.send({ model: 'attacker', input: 'Replaced input' });
expect(result.status).toBe(200);
expect(result.body).toEqual({
tenant: null,
userTenant: null,
userId,
admitted: responseId,
body: { model: 'agent', input: 'Saved input', background: true },
});
expect(f.authorize).toHaveBeenCalledTimes(1);
expect(f.onError).not.toHaveBeenCalled();
const persisted = await methods.findUser({ _id: userId });
expect(persisted?.tenantId).toBe(user);
expect(createBackgroundResponseEnvelope(record).principal.tenantId).toBeUndefined();
const resumeRecord: Parameters<typeof mintResponseResumeWorkerToken>[0] = {
...record,
generationCreatedAt: 123,
resume: {
intentId: 'intent',
actionId: 'action',
previousProviderExecutionId: 'previous',
providerExecutionId: 'provider',
fingerprint: 'fingerprint',
resolutionJSON: '{}',
state: 'reserved',
reservedAt: 1,
},
};
const resume = mintResponseResumeWorkerToken(resumeRecord, secret);
expect(createResponseResumeEnvelope(resumeRecord).principal.tenantId).toBeUndefined();
expect(jwt.decode(resume)).not.toHaveProperty('tenantId');
},
);
it('preserves named partitions and rejects a different request tenant before database access', async () => {
const record = await fixture('tenant-a');
const token = mintBackgroundResponseWorkerToken(record, secret);
const own = app('tenant-a');
expect(
(
await request(own.server)
.post('/')
.set('x-lc-response-worker', '1')
.auth(token, { type: 'bearer' })
).status,
).toBe(200);
const other = app('tenant-b');
expect(
(
await request(other.server)
.post('/')
.set('x-lc-response-worker', '1')
.auth(token, { type: 'bearer' })
).status,
).toBe(401);
expect(other.findUser).not.toHaveBeenCalled();
expect(other.authorize).not.toHaveBeenCalled();
});
it.each(['', '_', '__SYSTEM__'])('rejects noncanonical wire tenant %p', async (tenantId) => {
await fixture();
const token = jwt.sign(
{ purpose: 'responses_background', responseId, userId, tenantId, sourceKeyId: 'original-key' },
secret,
{ algorithm: 'HS256', audience: 'librechat-responses-worker', expiresIn: '60s' },
);
const f = app();
expect(
(
await request(f.server)
.post('/')
.set('x-lc-response-worker', '1')
.auth(token, { type: 'bearer' })
).status,
).toBe(401);
expect(f.findUser).not.toHaveBeenCalled();
expect(f.authorize).not.toHaveBeenCalled();
});
it.each(['_', '__SYSTEM__'])(
'refuses to mint execution authority for internal partition %p',
async (tenantId) => {
const record = await fixture();
expect(() => mintBackgroundResponseWorkerToken({ ...record, tenantId }, secret)).toThrow();
},
);
it.each(['responses_background', 'responses_resume'])(
'reports only verified expiration as renewable for %s credentials',
async (purpose) => {
const record = await fixture();
const token = jwt.sign(
{
purpose,
responseId: record.responseId,
userId,
sourceKeyId: record.sourceKeyId,
...(purpose === 'responses_resume'
? { intentId: 'intent', generationCreatedAt: 123, providerExecutionId: 'provider' }
: {}),
},
secret,
{ algorithm: 'HS256', audience: 'librechat-responses-worker', expiresIn: -1 },
);
const f = app();
const result = await request(f.server)
.post('/')
.set('x-lc-response-worker', '1')
.set('Authorization', `Bearer ${token}`)
.send({});
expect(result.status).toBe(401);
expect(result.body.error.code).toBe(RESPONSE_WORKER_TOKEN_EXPIRED);
expect(f.findUser).not.toHaveBeenCalled();
expect(f.authorize).not.toHaveBeenCalled();
expect(f.onError).not.toHaveBeenCalled();
},
);
it.each(['signature', 'audience', 'purpose', 'claims'])(
'keeps an expired token with invalid %s a permanent authentication denial',
async (invalid) => {
const record = await fixture();
const token = jwt.sign(
{
purpose: invalid === 'purpose' ? 'other-worker' : 'responses_background',
responseId: invalid === 'claims' ? 42 : record.responseId,
userId,
sourceKeyId: record.sourceKeyId,
},
invalid === 'signature' ? 'wrong-secret' : secret,
{
algorithm: 'HS256',
audience: invalid === 'audience' ? 'wrong-audience' : 'librechat-responses-worker',
expiresIn: -1,
},
);
const f = app();
const result = await request(f.server)
.post('/')
.set('x-lc-response-worker', '1')
.set('Authorization', `Bearer ${token}`)
.send({});
expect(result.status).toBe(401);
expect(result.body.error.code).toBe('invalid_response_worker');
expect(f.findUser).not.toHaveBeenCalled();
expect(f.authorize).not.toHaveBeenCalled();
},
);
it.each([true, false])(
'retries an expired delivery credential through real HTTP without bypassing current source policy: allowed=%s',
async (allowed) => {
const record = await fixture();
let admissions = 0;
const f = app(undefined, async (req, res) => {
const admitted = getBackgroundResponseWorkerRecord(req);
if (!admitted) throw new Error('Worker admission missing');
expect(req.body.input).toBe('Saved input');
const response = {
...buildPendingBackgroundResponse(admitted),
status: 'in_progress' as const,
};
const saved = await storage.publish(admitted, admitted.responseId, admitted.revision, {
status: 'in_progress',
generationCreatedAt: 1234,
responseJSON: JSON.stringify(response),
});
if (!saved.ok) throw new Error('Fixture enrollment failed');
admissions++;
res.status(202).json(response);
});
f.authorize.mockResolvedValue(allowed);
let clock = Date.now();
const clockSpy = jest.spyOn(Date, 'now').mockImplementation(() => clock);
const tokens = new Set<string>();
let attempts = 0;
const transport = createBackgroundResponseTransport({
storage,
getOrigin: () => 'http://127.0.0.1:3088',
signingSecret: () => secret,
acknowledgementTimeoutMs: async () => 120000,
fetch: async (_input, init) => {
attempts++;
const headers = Object.fromEntries(new Headers(init?.headers).entries());
tokens.add(headers.authorization);
if (attempts === 1) clock += 61000;
const result = await request(f.server).post('/').set(headers).send({});
return Response.json(result.body, { status: result.status });
},
});
try {
await expect(transport(record)).rejects.toMatchObject({
code: RESPONSE_WORKER_TOKEN_EXPIRED,
certainty: 'definite',
retryable: true,
});
expect(admissions).toBe(0);
expect(f.authorize).not.toHaveBeenCalled();
if (allowed) {
await expect(transport(record)).resolves.toEqual({
status: 'started',
generationCreatedAt: 1234,
});
expect(admissions).toBe(1);
} else {
await expect(transport(record)).rejects.toMatchObject({
code: 'response_admission_rejected',
status: 401,
retryable: false,
});
expect(admissions).toBe(0);
expect((await storage.read({ userId }, responseId))?.generationCreatedAt).toBeUndefined();
}
expect(f.authorize).toHaveBeenCalledTimes(1);
expect(tokens.size).toBe(2);
} finally {
clockSpy.mockRestore();
}
},
);

View file

@ -0,0 +1,267 @@
import { z } from 'zod';
import jwt from 'jsonwebtoken';
import { getTenantId, tenantStorage } from '@librechat/data-schemas';
import type { IUser, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas';
import type { Request, RequestHandler } from 'express';
import type { ResumedAgentExecutionIdentity } from '../remote/lifecycle';
import type { ResponseResumeAdmissionResult } from './types';
import type { ContextRequest } from '~/middleware/tenant';
import type { ServerRequest } from '~/types/http';
import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints';
import { responseTenantId, isResponseTenantId } from './scope';
import { tenantContextMiddleware } from '~/middleware/tenant';
const audience = 'librechat-responses-worker';
const purpose = 'responses_background';
const resumePurpose = 'responses_resume';
export const RESPONSE_WORKER_TOKEN_EXPIRED = 'response_worker_token_expired';
// Per-attempt security window, not a limit on durable queue work; expiration renews the credential.
const workerTokenLifetime = '60s';
const baseClaims = z.object({
responseId: z.string().min(1),
userId: z.string().min(1),
tenantId: z.string().min(1).refine(isResponseTenantId).optional(),
sourceKeyId: z.string().min(1),
});
const claimsSchema = z.discriminatedUnion('purpose', [
baseClaims.extend({ purpose: z.literal(purpose) }),
baseClaims.extend({
purpose: z.literal(resumePurpose),
intentId: z.string().min(1).max(256),
generationCreatedAt: z.number().int().positive(),
providerExecutionId: z.string().min(1).max(128),
}),
]);
type WorkerClaims = z.infer<typeof claimsSchema>;
type WorkerTokenError = 'invalid_response_worker' | typeof RESPONSE_WORKER_TOKEN_EXPIRED;
type WorkerTokenResult =
| { ok: true; value: WorkerClaims }
| { ok: false; error: { code: WorkerTokenError } };
function verifyWorkerToken(token: string, secret: string): WorkerTokenResult {
const verification: jwt.VerifyOptions = { algorithms: ['HS256'], audience };
try {
const parsed = claimsSchema.safeParse(jwt.verify(token, secret, verification));
if (parsed.success) return { ok: true, value: parsed.data };
} catch (error) {
if (error instanceof jwt.TokenExpiredError) {
try {
// verify() checks expiration before audience. Revalidate every other constraint before
// treating this as a renewable credential rather than a forged or malformed request.
const expired = claimsSchema.safeParse(
jwt.verify(token, secret, {
...verification,
ignoreExpiration: true,
}),
);
if (expired.success) return { ok: false, error: { code: RESPONSE_WORKER_TOKEN_EXPIRED } };
} catch {
// Invalid signatures, audiences, purposes and claim shapes remain permanent denials.
}
}
}
return { ok: false, error: { code: 'invalid_response_worker' } };
}
interface WorkerAdmission {
record: StoredResponseRecord;
storage: ResponseStorage;
interactive: boolean;
resume?: ResumedAgentExecutionIdentity & { intentId: string; checkpointNamespace: string };
}
const admitted = new WeakMap<Request, WorkerAdmission>();
export interface BackgroundResponseWorkerDependencies {
storage: ResponseStorage;
signingSecret: () => string;
findUser: (userId: string) => Promise<IUser | null>;
/** Check original key existence/expiry, current principal, and resolved auth/Agent policy. */
authorize: (record: StoredResponseRecord, user: IUser) => Promise<boolean>;
/** Install only alongside a capable resume executor and complete current checkpoint validation. */
admitResume?: (
record: StoredResponseRecord,
user: IUser,
intentId: string,
) => Promise<ResponseResumeAdmissionResult>;
onError: (error: unknown) => void;
}
/** Distinct purpose is rejected by older initial-execution workers, not silently stripped. */
export function mintResponseResumeWorkerToken(
record: StoredResponseRecord,
secret: string,
): string {
if (
!record.sourceKeyId ||
!secret ||
!record.resume ||
record.generationCreatedAt == null ||
!isResponseTenantId(record.tenantId)
)
throw new Error('Response resume worker authority unavailable');
return jwt.sign(
{
purpose: resumePurpose,
responseId: record.responseId,
userId: record.userId,
...(responseTenantId(record.tenantId) == null
? {}
: { tenantId: responseTenantId(record.tenantId) }),
sourceKeyId: record.sourceKeyId,
intentId: record.resume.intentId,
generationCreatedAt: record.generationCreatedAt,
providerExecutionId: record.resume.providerExecutionId,
},
secret,
{ algorithm: 'HS256', audience, expiresIn: workerTokenLifetime },
);
}
/** A worker credential is server-only, short-lived, and bound to a single API-key-authenticated response. */
export function mintBackgroundResponseWorkerToken(
record: StoredResponseRecord,
secret: string,
): string {
if (!record.sourceKeyId || !secret || !isResponseTenantId(record.tenantId))
throw new Error('Background response worker authority unavailable');
return jwt.sign(
{
purpose,
responseId: record.responseId,
userId: record.userId,
...(responseTenantId(record.tenantId) == null
? {}
: { tenantId: responseTenantId(record.tenantId) }),
sourceKeyId: record.sourceKeyId,
},
secret,
{ algorithm: 'HS256', audience, expiresIn: workerTokenLifetime },
);
}
export function getBackgroundResponseWorkerRecord(req: Request): StoredResponseRecord | undefined {
return admitted.get(req)?.record;
}
export function getBackgroundResponseWorkerAdmission(req: Request): WorkerAdmission | undefined {
return admitted.get(req);
}
/** Installed only on Responses ingress. Ordinary bearer authentication stays unchanged. */
export function createBackgroundResponseWorkerAuth(
deps: BackgroundResponseWorkerDependencies,
ordinaryAuth: RequestHandler,
): RequestHandler {
return async (req, res, next) => {
if (req.headers['x-lc-response-worker'] !== '1') return ordinaryAuth(req, res, next);
const deny = (code: WorkerTokenError = 'invalid_response_worker') => {
res.status(401).json({ error: { code, message: 'Unauthorized' } });
};
if (req.method !== 'POST' || req.path !== '/') return deny();
const header = req.headers.authorization;
if (!header?.startsWith('Bearer ')) return deny();
let claims: WorkerClaims;
try {
const verified = verifyWorkerToken(header.slice(7), deps.signingSecret());
if (!verified.ok) return deny(verified.error.code);
claims = verified.value;
} catch {
return deny();
}
const contextTenant = getTenantId();
if (
responseTenantId(contextTenant) != null &&
responseTenantId(contextTenant) !== claims.tenantId
)
return deny();
try {
await tenantStorage.run({ tenantId: claims.tenantId, userId: claims.userId }, async () => {
const [loaded, user] = await Promise.all([
deps.storage.read(claims, claims.responseId),
deps.findUser(claims.userId),
]);
let record = loaded;
if (
!record ||
!user ||
String(user._id) !== record.userId ||
!isResponseTenantId(user.tenantId) ||
!isResponseTenantId(record.tenantId) ||
responseTenantId(user.tenantId) !== claims.tenantId ||
responseTenantId(record.tenantId) !== claims.tenantId ||
record.sourceKeyId !== claims.sourceKeyId ||
!(await deps.authorize(record, user))
)
return deny();
// Initial delivery replay only attaches. Never let an expired generation claim
// turn an already-enrolled (possibly paused/resumed) response into a replacement job.
if (claims.purpose === purpose && record.generationCreatedAt != null) {
res.status(409).json({ code: 'RUN_ALREADY_ADMITTED' });
return;
}
let resume: WorkerAdmission['resume'];
if (claims.purpose === resumePurpose) {
const original = record;
if (
!deps.admitResume ||
record.generationCreatedAt !== claims.generationCreatedAt ||
record.resume?.intentId !== claims.intentId ||
record.resume.providerExecutionId !== claims.providerExecutionId
)
return deny();
const result = await deps.admitResume(record, user, claims.intentId);
if (!result.ok) return deny();
record = result.record;
if (
record.responseId !== claims.responseId ||
record.userId !== claims.userId ||
!isResponseTenantId(record.tenantId) ||
responseTenantId(record.tenantId) !== claims.tenantId ||
record.sourceKeyId !== claims.sourceKeyId ||
record.generationCreatedAt !== claims.generationCreatedAt ||
record.agentId !== original.agentId ||
record.conversationId !== original.conversationId ||
record.requestJSON !== original.requestJSON ||
record.resume?.intentId !== claims.intentId ||
record.resume.providerExecutionId !== claims.providerExecutionId ||
record.resume.state !== 'admitted' ||
record.status !== 'in_progress' ||
record.cancellationRequestedAt != null ||
!result.checkpointNamespace.startsWith(
checkpointOwnerNamespacePrefix(record.userId, record.tenantId),
) ||
!(await deps.authorize(record, user))
)
return deny();
resume = {
intentId: claims.intentId,
generationCreatedAt: claims.generationCreatedAt,
providerExecutionId: claims.providerExecutionId,
checkpointNamespace: result.checkpointNamespace,
};
}
const request = req as ServerRequest & Pick<ContextRequest, 'tenantId'>;
// findUser supplies a lean auth DTO; do not mutate its cached tenant representation.
request.user = Object.assign({}, user, { id: record.userId, tenantId: claims.tenantId });
request.tenantId = claims.tenantId;
// Caller body/headers cannot replace the persisted, already validated request.
request.body = JSON.parse(record.requestJSON);
admitted.set(req, {
record,
storage: deps.storage,
interactive: deps.admitResume != null,
...(resume ? { resume } : {}),
});
tenantContextMiddleware(request, res, next);
});
} catch (error) {
deps.onError(error);
res.status(503).json({
error: {
code: 'response_authority_unavailable',
message: 'Response authority is unavailable.',
},
});
}
};
}

View file

@ -3129,6 +3129,7 @@ class GenerationJobManagerClass {
tenantId: jobData.tenantId,
conversationId: jobData.conversationId,
checkpointNamespace: jobData.checkpointNamespace,
responseCheckpointConfigJSON: jobData.responseCheckpointConfigJSON,
generationProtocolVersion: jobData.generationProtocolVersion,
earlyBufferOverflow: jobData.earlyBufferOverflow,
userMessage: jobData.userMessage,

View file

@ -338,6 +338,7 @@ describe('RedisJobStore', () => {
compact: true,
isTemporary: false,
retentionExpiresAt: '2030-01-01T00:00:00.000Z',
responseCheckpointConfigJSON: '{"type":"mongo","checkpointCollectionName":"responses"}',
agentEventDeliveryKey: 'completion-delivery-1',
agentEventInvocationKey: 'original-delivery-1',
agentEventInvocationGenerationCreatedAt: 987654,
@ -457,6 +458,7 @@ describe('RedisJobStore', () => {
agent_id: 'agent-1',
isTemporary: false,
retentionExpiresAt: '2030-01-01T00:00:00.000Z',
responseCheckpointConfigJSON: '{"type":"mongo","checkpointCollectionName":"responses"}',
scheduleId: 'schedule-1',
scheduleMCPCompletion: {
scheduleId: 'original-schedule',
@ -526,6 +528,7 @@ describe('RedisJobStore', () => {
agent_id: 'agent-1',
isTemporary: '0',
retentionExpiresAt: '2030-01-01T00:00:00.000Z',
responseCheckpointConfigJSON: '{"type":"mongo","checkpointCollectionName":"responses"}',
scheduleId: 'schedule-1',
scheduledFor: '2026-08-17T12:00:00.000Z',
scheduleConfigRevision: '4',

View file

@ -5774,6 +5774,7 @@ export class RedisJobStore implements IJobStoreV2 {
agent_id: data.agent_id || undefined,
isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,
retentionExpiresAt: data.retentionExpiresAt || undefined,
responseCheckpointConfigJSON: data.responseCheckpointConfigJSON || undefined,
agentEventDeliveryKey: data.agentEventDeliveryKey || undefined,
agentEventInvocationKey: data.agentEventInvocationKey || undefined,
agentEventInvocationGenerationCreatedAt: data.agentEventInvocationGenerationCreatedAt

View file

@ -129,6 +129,7 @@ export interface SerializableJobData {
* `checkpoint_ns`, which the saver adapter maps to this storage scope.
* Legacy paused jobs omit it and use the historical unscoped storage. */
checkpointNamespace?: string;
responseCheckpointConfigJSON?: string;
completedAt?: number;
conversationId?: string;
error?: string;
@ -512,6 +513,7 @@ export type JobMetadataPatch = Partial<
| 'agent_id'
| 'isTemporary'
| 'retentionExpiresAt'
| 'responseCheckpointConfigJSON'
| 'agentEventDeliveryKey'
| 'agentEventInvocationKey'
| 'agentEventInvocationGenerationCreatedAt'

View file

@ -29,6 +29,8 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): J
patch.terminalPersistenceOwner = parseTerminalPersistenceOwner(
metadata.terminalPersistenceOwner,
);
if (metadata.responseCheckpointConfigJSON !== undefined)
patch.responseCheckpointConfigJSON = metadata.responseCheckpointConfigJSON;
if (metadata.responseMessageId) {
patch.responseMessageId = metadata.responseMessageId;
}

View file

@ -24,6 +24,8 @@ export interface GenerationJobMetadata {
/** Immutable per-generation saver scope. LangGraph's root `checkpoint_ns`
* remains empty; the checkpointer adapter maps this scope into storage. */
checkpointNamespace?: string;
/** Server-pinned saver settings retained for expired background-response cleanup. */
responseCheckpointConfigJSON?: string;
/** Immutable generation protocol. Missing on legacy records means v1. */
generationProtocolVersion?: 1 | 2;
earlyBufferOverflow?: EarlyBufferOverflowState;

View file

@ -15,6 +15,7 @@ export { createResponseStorage } from './methods/response';
export type {
ResponseOwner,
ResponseStorage,
ResponseTerminalGuard,
ResponseCleanupIdentity,
ResponseReservation,
ResponseReservationResult,

View file

@ -1517,3 +1517,101 @@ it('retries a failed initial ID assignment without reopening its unused source s
await storage.reserve(reservation({ conversationId: input.newConversationId })),
).toMatchObject({ ok: true });
}));
it('stages one exact execution receipt independently of repeated projection revisions', () =>
scoped(async () => {
await storage.reserve(reservation({ sourceKeyId: 'original-key' }));
const running = await storage.publish(owner, 'resp_one', 0, {
status: 'in_progress',
generationCreatedAt: 10,
responseJSON: '{}',
});
if (!running.ok) throw new Error('Publication failed');
for (let i = 0; i < 30; i++) {
const current = (await storage.read(owner, 'resp_one'))!;
expect(
(
await storage.publish(owner, 'resp_one', current.revision, {
status: 'in_progress',
responseJSON: JSON.stringify({ progress: i }),
})
).ok,
).toBe(true);
}
expect(
await storage.stageTerminal(owner, 'resp_one', running.record.revision, outcome),
).toEqual({ ok: false, code: 'response_conflict' });
const attempts = await Promise.all(
Array.from({ length: 12 }, () =>
storage.stageTerminal(owner, 'resp_one', { execution: running.record }, outcome),
),
);
expect(attempts.every((result) => result.ok)).toBe(true);
const staged = (await storage.read(owner, 'resp_one'))!;
expect(staged.pendingTerminal).toEqual(outcome);
expect(staged.revision).toBe(running.record.revision + 31);
expect(
await storage.stageTerminal(
owner,
'resp_one',
{ execution: running.record },
{
...outcome,
ownerId: 'another-worker',
},
),
).toEqual({ ok: false, code: 'response_conflict' });
expect((await storage.finishTerminal(owner, 'resp_one', staged.revision, outcome)).ok).toBe(
true,
);
}));
it.each(['agentId', 'conversationId', 'sourceKeyId', 'generationCreatedAt'] as const)(
'does not stage a retained execution with a mismatched %s',
(field) =>
scoped(async () => {
await storage.reserve(reservation({ sourceKeyId: 'original-key' }));
const running = await storage.publish(owner, 'resp_one', 0, {
status: 'in_progress',
generationCreatedAt: 10,
responseJSON: '{}',
});
if (!running.ok) throw new Error('Publication failed');
const execution = {
...running.record,
[field]: field === 'generationCreatedAt' ? 11 : 'other',
};
expect(await storage.stageTerminal(owner, 'resp_one', { execution }, outcome)).toEqual({
ok: false,
code: 'response_conflict',
});
expect((await storage.read(owner, 'resp_one'))?.pendingTerminal).toBeUndefined();
}),
);
it('matches the complete reviewed and resumed execution binding rather than its projection', () =>
scoped(async () => {
await pause();
await storage.reserveResume(owner, 'resp_one', resume(), 100);
await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101);
const retained = (await storage.read(owner, 'resp_one'))!;
const receipt = { ...outcome, providerExecutionId: 'provider-two' };
for (const execution of [
{ ...retained, review: undefined },
{ ...retained, review: { ...retained.review!, checkpointId: 'different' } },
{ ...retained, resume: { ...retained.resume!, fingerprint: 'different' } },
{ ...retained, resume: { ...retained.resume!, state: 'reserved' as const } },
])
expect(await storage.stageTerminal(owner, 'resp_one', { execution }, receipt)).toEqual({
ok: false,
code: 'response_conflict',
});
const current = await storage.publish(owner, 'resp_one', retained.revision, {
status: 'in_progress',
responseJSON: '{"projection":"new"}',
});
expect(current.ok).toBe(true);
expect(
(await storage.stageTerminal(owner, 'resp_one', { execution: retained }, receipt)).ok,
).toBe(true);
}));

View file

@ -77,11 +77,71 @@ export type ResponseResumeResult =
| { ok: true; record: StoredResponseRecord; replayed: boolean }
| { ok: false; code: 'resume_conflict' };
/** A drained execution's authority is independent of polling's projection revision. */
export type ResponseTerminalGuard =
| number
| {
execution: Pick<
StoredResponseRecord,
'agentId' | 'sourceKeyId' | 'conversationId' | 'generationCreatedAt' | 'review' | 'resume'
>;
};
function terminalExecutionFilter(guard: Exclude<ResponseTerminalGuard, number>) {
const { execution } = guard;
const filter: Record<string, unknown> = {
agentId: execution.agentId,
sourceKeyId: execution.sourceKeyId ?? { $exists: false },
conversationId: execution.conversationId,
generationCreatedAt: execution.generationCreatedAt,
};
const review = execution.review;
if (review) {
const fields: Record<
keyof ResponseReviewCheckpoint,
ResponseReviewCheckpoint[keyof ResponseReviewCheckpoint]
> = {
actionId: review.actionId,
providerExecutionId: review.providerExecutionId,
expiresAt: review.expiresAt,
partialResponseJSON: review.partialResponseJSON,
visibleOutputTokens: review.visibleOutputTokens,
pendingActionJSON: review.pendingActionJSON,
checkpointConfigJSON: review.checkpointConfigJSON,
checkpointId: review.checkpointId,
checkpointNs: review.checkpointNs,
};
for (const [key, value] of Object.entries(fields))
filter[`review.${key}`] = value ?? { $exists: false };
} else filter.review = { $exists: false };
const resume = execution.resume;
if (resume) {
const fields: Record<
keyof ResponseResumeIntent,
ResponseResumeIntent[keyof ResponseResumeIntent]
> = {
intentId: resume.intentId,
actionId: resume.actionId,
previousProviderExecutionId: resume.previousProviderExecutionId,
providerExecutionId: resume.providerExecutionId,
fingerprint: resume.fingerprint,
resolutionJSON: resume.resolutionJSON,
decisionJSON: resume.decisionJSON,
state: resume.state,
reservedAt: resume.reservedAt,
admittedAt: resume.admittedAt,
};
for (const [key, value] of Object.entries(fields))
filter[`resume.${key}`] = value ?? { $exists: false };
} else filter.resume = { $exists: false };
return filter;
}
export interface ResponseStorage {
stageTerminal(
owner: ResponseOwner,
responseId: string,
revision: number,
guard: ResponseTerminalGuard,
receipt: ResponseTerminalReceipt,
): Promise<ResponseMutationResult>;
finishTerminal(
@ -183,7 +243,13 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
$and: [activeExpirationFilter<StoredResponseRecord>()],
});
return {
async stageTerminal(owner, responseId, revision, receipt) {
async stageTerminal(owner, responseId, guard, receipt) {
if (
typeof guard !== 'number' &&
guard.execution.generationCreatedAt !== receipt.generationCreatedAt
)
return { ok: false, code: 'response_conflict' };
const execution = typeof guard === 'number' ? undefined : terminalExecutionFilter(guard);
const pendingTerminal: ResponseTerminalReceipt = {
generationCreatedAt: receipt.generationCreatedAt,
providerExecutionId: receipt.providerExecutionId,
@ -202,7 +268,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
{
...scope(owner),
responseId,
revision,
...(execution ?? { revision: guard }),
generationCreatedAt: receipt.generationCreatedAt,
status: { $in: active },
pendingTerminal: { $exists: false },
@ -223,7 +289,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp
)
.lean();
if (record) return { ok: true, record };
const existing = await model.findOne({ ...scope(owner), responseId }).lean();
const existing = await model.findOne({ ...scope(owner), responseId, ...execution }).lean();
if (
existing?.generationCreatedAt === receipt.generationCreatedAt &&
existing.pendingTerminal &&