From d3fc693ccecc376b46a030da67b3a9361c8880ea Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:03:59 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A6=20feat:=20Dispatch=20and=20Recover?= =?UTF-8?q?=20Background=20Responses=20(#17600)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚦 feat: Dispatch and Recover Background Responses * fix: guard response publication and retirement by durable identity * fix: normalize legacy response tenant identity at execution boundaries * style: order response scope imports * style: Format Restacked Metadata Fields * test: Enroll Ordinary Abort Without External Ownership * test: Omit Ordinary Terminal Ownership Option * fix: Keep Terminal Reads and Worker Retries Under Their Owners --- .../src/agents/responses/authority.spec.ts | 113 ++++ .../api/src/agents/responses/authority.ts | 90 +++ .../src/agents/responses/background.spec.ts | 509 ++++++++++++++ .../api/src/agents/responses/background.ts | 213 ++++++ .../api/src/agents/responses/checkpoint.ts | 49 ++ .../src/agents/responses/completion.spec.ts | 141 ++++ .../api/src/agents/responses/completion.ts | 482 ++++++++------ .../api/src/agents/responses/foreground.ts | 190 +++++- packages/api/src/agents/responses/index.ts | 14 +- .../api/src/agents/responses/queue.spec.ts | 449 +++++++++++++ packages/api/src/agents/responses/queue.ts | 240 +++++++ .../api/src/agents/responses/recovery.spec.ts | 630 ++++++++++++++++++ .../api/src/agents/responses/release.spec.ts | 143 ++++ .../src/agents/responses/retirement.spec.ts | 374 +++++++++++ .../api/src/agents/responses/retirement.ts | 256 +++++++ packages/api/src/agents/responses/scope.ts | 14 + .../src/agents/responses/transport.spec.ts | 172 +++++ .../api/src/agents/responses/transport.ts | 189 ++++++ .../api/src/agents/responses/worker.spec.ts | 352 ++++++++++ packages/api/src/agents/responses/worker.ts | 267 ++++++++ .../api/src/stream/GenerationJobManager.ts | 1 + .../stream/__tests__/RedisJobStore.spec.ts | 3 + .../stream/implementations/RedisJobStore.ts | 1 + .../api/src/stream/interfaces/IJobStore.ts | 2 + packages/api/src/stream/metadata.ts | 2 + packages/api/src/types/stream.ts | 2 + packages/data-schemas/src/index.ts | 1 + .../data-schemas/src/methods/response.spec.ts | 98 +++ packages/data-schemas/src/methods/response.ts | 74 +- 29 files changed, 4872 insertions(+), 199 deletions(-) create mode 100644 packages/api/src/agents/responses/authority.spec.ts create mode 100644 packages/api/src/agents/responses/authority.ts create mode 100644 packages/api/src/agents/responses/background.spec.ts create mode 100644 packages/api/src/agents/responses/background.ts create mode 100644 packages/api/src/agents/responses/checkpoint.ts create mode 100644 packages/api/src/agents/responses/completion.spec.ts create mode 100644 packages/api/src/agents/responses/queue.spec.ts create mode 100644 packages/api/src/agents/responses/queue.ts create mode 100644 packages/api/src/agents/responses/recovery.spec.ts create mode 100644 packages/api/src/agents/responses/release.spec.ts create mode 100644 packages/api/src/agents/responses/retirement.spec.ts create mode 100644 packages/api/src/agents/responses/retirement.ts create mode 100644 packages/api/src/agents/responses/scope.ts create mode 100644 packages/api/src/agents/responses/transport.spec.ts create mode 100644 packages/api/src/agents/responses/transport.ts create mode 100644 packages/api/src/agents/responses/worker.spec.ts create mode 100644 packages/api/src/agents/responses/worker.ts diff --git a/packages/api/src/agents/responses/authority.spec.ts b/packages/api/src/agents/responses/authority.spec.ts new file mode 100644 index 0000000000..72b5ceb2ca --- /dev/null +++ b/packages/api/src/agents/responses/authority.spec.ts @@ -0,0 +1,113 @@ +import { tenantStorage } from '@librechat/data-schemas'; +import { PermissionBits, Permissions, PermissionTypes } from 'librechat-data-provider'; +import type { BackgroundResponseAuthorityDependencies } from './authority'; +import { createBackgroundResponseAuthority } from './authority'; + +const owner = { userId: 'background-owner', tenantId: 'background-tenant' }; +function fixture(options: { tenantId?: string | null } = { tenantId: owner.tenantId }) { + const user = { + _id: owner.userId, + role: 'USER', + tenantId: options.tenantId, + provider: 'local', + twoFactorEnabled: false, + }; + const deps: BackgroundResponseAuthorityDependencies = { + findUser: jest.fn().mockResolvedValue(user), + getKey: jest.fn().mockResolvedValue({}), + getRole: jest.fn().mockResolvedValue({ + permissions: { [PermissionTypes.REMOTE_AGENTS]: { [Permissions.USE]: true } }, + }), + getConfig: jest.fn().mockResolvedValue({ + endpoints: { + agents: { + remoteApi: { background: { enabled: true }, auth: { apiKey: { enabled: true } } }, + }, + }, + }), + getAgentPermissions: jest.fn().mockResolvedValue(PermissionBits.VIEW), + isPrincipalActive: jest.fn().mockResolvedValue(true), + }; + return { deps, user, authorize: createBackgroundResponseAuthority(deps) }; +} +const originalPolicy = process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION; +afterEach(() => { + if (originalPolicy === undefined) delete process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION; + else process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = originalPolicy; +}); + +it('rechecks shared enrollment policy when enforcement changes after queue admission', async () => { + const { user, deps, authorize } = fixture(); + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'false'; + expect(await authorize(owner, 'agent', 'original-key')).toBe(true); + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true'; + jest.mocked(deps.getKey).mockClear(); + expect(await authorize(owner, 'agent', 'original-key')).toBe(false); + expect(deps.getKey).not.toHaveBeenCalled(); + user.twoFactorEnabled = true; + expect(await authorize(owner, 'agent', 'original-key')).toBe(true); +}); + +it('uses an already-loaded user while enforcing enrollment before background grants', async () => { + const { deps, authorize } = fixture(); + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true'; + const user = await deps.findUser(owner.userId); + if (!user) throw new Error('Fixture missing user'); + jest.mocked(deps.findUser).mockClear(); + expect( + await tenantStorage.run(owner, () => authorize(owner, 'agent', 'original-key', user)), + ).toBe(false); + expect(deps.findUser).not.toHaveBeenCalled(); + expect(deps.getKey).not.toHaveBeenCalled(); +}); + +it('accepts the host enrollment policy dependency without bypassing it for a loaded user', async () => { + const { deps } = fixture(); + deps.enrollmentRequired = jest.fn().mockReturnValue(true); + const user = await deps.findUser(owner.userId); + if (!user) throw new Error('Fixture missing user'); + expect(await createBackgroundResponseAuthority(deps)(owner, 'agent', 'original-key', user)).toBe( + false, + ); + expect(deps.enrollmentRequired).toHaveBeenCalledWith(user); + expect(deps.getKey).not.toHaveBeenCalled(); +}); + +it('preserves federated policy exceptions and tenant isolation', async () => { + const { user, authorize } = fixture(); + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true'; + user.provider = 'openid'; + expect(await authorize(owner, 'agent', 'original-key')).toBe(true); + expect( + await tenantStorage.run({ tenantId: 'other-tenant' }, () => + authorize(owner, 'agent', 'original-key'), + ), + ).toBe(false); +}); + +it.each([undefined, null, ''])( + 'accepts a legacy tenantless user field %p without changing named authority', + async (tenantId) => { + const { deps, authorize } = fixture({ tenantId }); + deps.enrollmentRequired = () => false; + const principal = { userId: owner.userId }; + expect(await tenantStorage.run(principal, () => authorize(principal, 'agent', 'key'))).toBe( + true, + ); + const loaded = await deps.findUser(principal.userId); + if (!loaded) throw new Error('Fixture missing user'); + jest.mocked(deps.findUser).mockClear(); + expect(await authorize({ ...principal, tenantId: '' }, 'agent', 'key', loaded)).toBe(true); + expect(deps.findUser).not.toHaveBeenCalled(); + expect(await authorize(owner, 'agent', 'key', loaded)).toBe(false); + }, +); + +it.each(['_', '__SYSTEM__'])( + 'rejects internal owner partition %p before principal lookup', + async (tenantId) => { + const { deps, authorize } = fixture({ tenantId }); + expect(await authorize({ ...owner, tenantId }, 'agent', 'key')).toBe(false); + expect(deps.findUser).not.toHaveBeenCalled(); + }, +); diff --git a/packages/api/src/agents/responses/authority.ts b/packages/api/src/agents/responses/authority.ts new file mode 100644 index 0000000000..36dd2cf3bb --- /dev/null +++ b/packages/api/src/agents/responses/authority.ts @@ -0,0 +1,90 @@ +import { PermissionBits, Permissions, PermissionTypes } from 'librechat-data-provider'; +import { getTenantId, SYSTEM_TENANT_ID, tenantStorage } from '@librechat/data-schemas'; +import type { + AppConfig, + IUser, + IRole, + ResponseOwner, + AgentApiKeyBinding, +} from '@librechat/data-schemas'; +import { isTwoFactorEnrollmentRequired } from '~/auth/twoFactor'; +import { responseTenantId, isResponseTenantId } from './scope'; + +export interface BackgroundResponseAuthorityDependencies { + enrollmentRequired?: typeof isTwoFactorEnrollmentRequired; + findUser: (userId: string) => Promise; + getKey: ( + keyId: string, + userId: string, + ) => Promise<{ expiresAt?: Date; binding?: AgentApiKeyBinding } | null>; + authorizeBinding?: ( + binding: AgentApiKeyBinding, + keyId: string, + owner: ResponseOwner, + agentId: string, + ) => Promise; + getRole: (role: string) => Promise; + getConfig: (user: IUser) => Promise; + getAgentPermissions: (agentId: string, user: { id: string; role: string }) => Promise; + isPrincipalActive: (userId: string) => Promise; + now?: () => number; +} + +/** Background work retains its original credential's authority, not the worker's signing key. */ +export function createBackgroundResponseAuthority(deps: BackgroundResponseAuthorityDependencies) { + return async ( + owner: ResponseOwner, + agentId: string, + sourceKeyId?: string, + loadedUser?: IUser, + ): Promise => { + if (!sourceKeyId || !isResponseTenantId(owner.tenantId)) return false; + const tenantId = responseTenantId(owner.tenantId); + const currentTenant = getTenantId(); + if ( + currentTenant != null && + currentTenant !== SYSTEM_TENANT_ID && + responseTenantId(currentTenant) !== tenantId + ) + return false; + return tenantStorage.run({ tenantId, userId: owner.userId }, async () => { + const user = loadedUser ?? (await deps.findUser(owner.userId)); + if ( + !user || + String(user._id) !== owner.userId || + !isResponseTenantId(user.tenantId) || + responseTenantId(user.tenantId) !== tenantId || + !user.role || + (deps.enrollmentRequired ?? isTwoFactorEnrollmentRequired)(user) + ) + return false; + const [key, role, config, active, permissions] = await Promise.all([ + deps.getKey(sourceKeyId, owner.userId), + deps.getRole(user.role), + deps.getConfig(user), + deps.isPrincipalActive(owner.userId), + deps.getAgentPermissions(agentId, { id: owner.userId, role: user.role }), + ]); + const remote = config.endpoints?.agents?.remoteApi; + if ( + key?.binding && + !(await deps.authorizeBinding?.( + key.binding, + sourceKeyId, + { userId: owner.userId, tenantId }, + agentId, + )) + ) + return false; + return Boolean( + key && + (!key.expiresAt || key.expiresAt.getTime() > (deps.now ?? Date.now)()) && + active && + role?.permissions?.[PermissionTypes.REMOTE_AGENTS]?.[Permissions.USE] === true && + remote?.background?.enabled === true && + remote?.auth?.apiKey?.enabled !== false && + (permissions & PermissionBits.VIEW) === PermissionBits.VIEW, + ); + }); + }; +} diff --git a/packages/api/src/agents/responses/background.spec.ts b/packages/api/src/agents/responses/background.spec.ts new file mode 100644 index 0000000000..ee47c4bd9c --- /dev/null +++ b/packages/api/src/agents/responses/background.spec.ts @@ -0,0 +1,509 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { createResponseStorage, tenantStorage, runAsSystem } from '@librechat/data-schemas'; +import type { + ResponseStorage, + StoredResponseRecord, + ResponseOwner, + ResponseTerminalReceipt, +} from '@librechat/data-schemas'; +import type { BackgroundResponseDependencies } from './background'; +import type { ResponsesRunEnvelope } from '../envelope'; +import type { Response } from './types'; +import { + buildAggregatedResponse, + createResponseAggregator, + createResponseContext, +} from './service'; +import { createBackgroundResponseService } from './background'; +import { createResponseCompletion } from './completion'; +import { createAgentRunEnvelope } from '../envelope'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +const owner = { userId: 'owner', tenantId: 'tenant-a' }; +const scope = (work: () => Promise) => tenantStorage.run({ tenantId: owner.tenantId }, work); +const envelope = (payload: Partial = {}) => + createAgentRunEnvelope({ + protocol: 'responses', + requestId: 'request', + receivedAt: 100, + principal: { id: owner.userId, tenantId: owner.tenantId }, + payload: { model: 'agent', input: 'Hello', background: true, store: true, ...payload }, + }); + +function completed(record: StoredResponseRecord): Response { + const aggregator = createResponseAggregator(); + aggregator.addText('# Persisted response\n\n**Full answer.**'); + return { + ...buildAggregatedResponse( + createResponseContext({ model: record.agentId, input: 'Hello' }, record.responseId), + aggregator, + ), + background: true, + store: true, + }; +} + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + storage = createResponseStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); +}); + +function fixture() { + const deliveries = new Map(); + const cleanupCheckpoint = jest.fn(async () => {}); + const releaseExecution = jest.fn(async () => {}); + const completion = createResponseCompletion({ storage, cleanupCheckpoint, releaseExecution }); + const deps: BackgroundResponseDependencies = { + storage, + authorize: async (principal, agentId) => + principal.userId === owner.userId && agentId === 'agent', + resolveContinuation: async () => ({ conversationId: 'conversation', parentMessageId: 'root' }), + enqueue: async (record) => { + deliveries.set(record.deliveryKey!, record); + }, + observe: async () => null, + recoverTerminal: completion.finish, + cancel: async () => {}, + now: () => 200, + }; + return { + deps, + deliveries, + completion, + cleanupCheckpoint, + releaseExecution, + service: createBackgroundResponseService(deps), + }; +} + +async function admit(record: StoredResponseRecord): Promise { + const result = await storage.publish(record, record.responseId, record.revision, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: JSON.stringify({ + ...completed(record), + status: 'in_progress', + completed_at: null, + output: [], + }), + }); + if (!result.ok) throw new Error('Fixture admission failed'); + return result.record; +} + +function terminalReceipt( + record: StoredResponseRecord, + status: ResponseTerminalReceipt['status'] = 'completed', +) { + if (record.generationCreatedAt == null) throw new Error('Fixture generation missing'); + const response: Response = { ...completed(record), status }; + const receipt: ResponseTerminalReceipt = { + generationCreatedAt: record.generationCreatedAt, + providerExecutionId: 'provider', + ownerId: 'worker', + status, + responseJSON: JSON.stringify(response), + checkpointNamespace: 'checkpoint', + }; + return { response, receipt }; +} + +it('reserves a generic response and preserves all validated execution settings', () => + scope(async () => { + const { service, deliveries } = fixture(); + const request = envelope({ + instructions: 'Keep instructions', + temperature: 0.4, + manualSkills: ['skill'], + max_output_tokens: 1234, + metadata: { purpose: 'test' }, + }); + const result = await service.create(request, 'stable-key'); + expect(result).toMatchObject({ + ok: true, + response: { status: 'queued', background: true, store: true, output: [] }, + }); + const record = [...deliveries.values()][0]; + expect(JSON.parse(record.requestJSON)).toEqual(request.payload); + expect(record.agentId).toBe('agent'); + })); + +it('replays the same identity, including after an uncertain queue publication', () => + scope(async () => { + const { service, deps, deliveries } = fixture(); + const enqueue = deps.enqueue; + let first = true; + deps.enqueue = async (record) => { + await enqueue(record); + if (first) { + first = false; + throw new Error('queue response lost'); + } + }; + await expect(service.create(envelope(), 'same')).rejects.toThrow('queue response lost'); + const result = await service.create(envelope(), 'same'); + expect(result.ok).toBe(true); + expect(deliveries.size).toBe(1); + const record = [...deliveries.values()][0]; + if (result.ok) expect(result.response.id).toBe(record.responseId); + })); + +it('canonicalizes nested request key order but rejects changed request values', () => + scope(async () => { + const { service, deliveries } = fixture(); + const first = await service.create(envelope({ metadata: { a: '1', b: '2' } }), 'same'); + const second = await service.create(envelope({ metadata: { b: '2', a: '1' } }), 'same'); + expect(second).toEqual(first); + expect(await service.create(envelope({ metadata: { a: 'changed', b: '2' } }), 'same')).toEqual({ + ok: false, + status: 409, + code: 'idempotency_conflict', + }); + expect(deliveries.size).toBe(1); + })); + +it('denies revoked authority before reserve, observation or cancellation', () => + scope(async () => { + const { service, deps, deliveries } = fixture(); + await service.create(envelope(), 'same'); + const record = [...deliveries.values()][0]; + deps.authorize = async () => false; + const observe = jest.spyOn(deps, 'observe'); + const cancel = jest.spyOn(deps, 'cancel'); + expect(await service.retrieve(owner, record.responseId)).toEqual({ + ok: false, + status: 404, + code: 'response_not_found', + }); + expect(await service.cancel(owner, record.responseId)).toEqual({ + ok: false, + status: 404, + code: 'response_not_found', + }); + expect(await service.create(envelope(), 'different')).toEqual({ + ok: false, + status: 403, + code: 'permission_denied', + }); + expect(observe).not.toHaveBeenCalled(); + expect(cancel).not.toHaveBeenCalled(); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + })); + +it.each(['completed', 'failed', 'incomplete', 'cancelled'] as const)( + 'defers an observed %s result until the terminal producer commits its journal', + (status) => + scope(async () => { + const f = fixture(); + await f.service.create(envelope(), 'same'); + const record = await admit([...f.deliveries.values()][0]); + const { response, receipt } = terminalReceipt(record, status); + const observe = jest + .spyOn(f.deps, 'observe') + .mockResolvedValue({ response, generationCreatedAt: 10 }); + const pending = await f.service.retrieve(owner, record.responseId); + expect(pending).toMatchObject({ ok: true, response: { status: 'in_progress', output: [] } }); + expect((await storage.read(owner, record.responseId))?.status).toBe('in_progress'); + expect(f.cleanupCheckpoint).not.toHaveBeenCalled(); + expect(f.releaseExecution).not.toHaveBeenCalled(); + await f.completion.stageAndFinish(record, receipt); + expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response }); + expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response }); + expect(observe).toHaveBeenCalledTimes(1); + expect(f.cleanupCheckpoint).toHaveBeenCalledTimes(1); + expect(f.releaseExecution).toHaveBeenCalled(); + }), +); + +it('persists cancellation intent before signalling and waits for the cancelled journal', () => + scope(async () => { + const f = fixture(); + await f.service.create(envelope(), 'same'); + const record = await admit([...f.deliveries.values()][0]); + f.deps.cancel = async (current) => { + expect((await storage.read(owner, current.responseId))?.cancellationRequestedAt).toBe(200); + }; + expect(await f.service.cancel(owner, record.responseId)).toMatchObject({ + ok: true, + response: { status: 'in_progress' }, + }); + const { response, receipt } = terminalReceipt(record, 'cancelled'); + f.deps.observe = async () => ({ response, generationCreatedAt: 10 }); + expect(await f.service.retrieve(owner, record.responseId)).toMatchObject({ + ok: true, + response: { status: 'in_progress' }, + }); + await f.completion.stageAndFinish(record, receipt); + expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response }); + })); + +it.each(['retrieve', 'replay', 'cancel'] as const)( + 'routes %s of a pending receipt through completion without restarting or cancelling work', + (entry) => + scope(async () => { + const f = fixture(); + await f.service.create(envelope(), 'same'); + const record = await admit([...f.deliveries.values()][0]); + const { response, receipt } = terminalReceipt(record); + const staged = await storage.stageTerminal( + owner, + record.responseId, + { execution: record }, + receipt, + ); + if (!staged.ok) throw new Error('Fixture staging failed'); + const enqueue = jest.spyOn(f.deps, 'enqueue'); + const cancel = jest.spyOn(f.deps, 'cancel'); + const observe = jest.spyOn(f.deps, 'observe'); + const actions = { + retrieve: () => f.service.retrieve(owner, record.responseId), + replay: () => f.service.create(envelope(), 'same'), + cancel: () => f.service.cancel(owner, record.responseId), + }; + const result = await actions[entry](); + expect(result).toEqual({ ok: true, response }); + expect(enqueue).not.toHaveBeenCalled(); + expect(cancel).not.toHaveBeenCalled(); + expect(observe).not.toHaveBeenCalled(); + expect(f.cleanupCheckpoint).toHaveBeenCalledTimes(1); + expect(f.releaseExecution).toHaveBeenCalledTimes(1); + expect( + (await storage.read(owner, record.responseId))?.cancellationRequestedAt, + ).toBeUndefined(); + }), +); + +it('recovers a journal that wins a progress CAS race and waits for cleanup before returning its result', () => + scope(async () => { + const f = fixture(); + await f.service.create(envelope(), 'same'); + const record = await admit([...f.deliveries.values()][0]); + const { response, receipt } = terminalReceipt(record); + f.deps.observe = async () => ({ + response: { ...response, status: 'in_progress' }, + generationCreatedAt: 10, + }); + let releaseCleanup!: () => void; + let enteredCleanup!: () => void; + const gate = new Promise((resolve) => { + releaseCleanup = resolve; + }); + const entered = new Promise((resolve) => { + enteredCleanup = resolve; + }); + f.cleanupCheckpoint.mockImplementation(async () => { + enteredCleanup(); + await gate; + }); + const publish = storage.publish.bind(storage); + const spy = jest.spyOn(storage, 'publish').mockImplementationOnce(async (...args) => { + const staged = await storage.stageTerminal( + owner, + record.responseId, + { execution: record }, + receipt, + ); + if (!staged.ok) throw new Error('Fixture staging race failed'); + return publish(...args); + }); + const read = f.service.retrieve(owner, record.responseId); + let settled = false; + void read.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + try { + await Promise.race([ + entered, + read.then(() => { + throw new Error('Read bypassed cleanup'); + }), + ]); + expect(settled).toBe(false); + expect(f.releaseExecution).not.toHaveBeenCalled(); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(receipt); + releaseCleanup(); + expect(await read).toEqual({ ok: true, response }); + expect(f.releaseExecution).toHaveBeenCalledTimes(1); + } finally { + releaseCleanup(); + await Promise.allSettled([read]); + spy.mockRestore(); + } + })); + +it('withholds terminal output through checkpoint and execution-release failures', () => + scope(async () => { + const f = fixture(); + await f.service.create(envelope(), 'same'); + const record = await admit([...f.deliveries.values()][0]); + const { response, receipt } = terminalReceipt(record); + const staged = await storage.stageTerminal( + owner, + record.responseId, + { execution: record }, + receipt, + ); + if (!staged.ok) throw new Error('Fixture staging failed'); + f.cleanupCheckpoint.mockRejectedValueOnce(new Error('checkpoint unavailable')); + await expect(f.service.retrieve(owner, record.responseId)).rejects.toThrow( + 'checkpoint unavailable', + ); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(receipt); + expect(f.releaseExecution).not.toHaveBeenCalled(); + f.releaseExecution.mockRejectedValueOnce(new Error('release unavailable')); + await expect(f.service.retrieve(owner, record.responseId)).rejects.toThrow( + 'release unavailable', + ); + expect((await storage.read(owner, record.responseId))?.status).toBe('completed'); + expect(await f.service.retrieve(owner, record.responseId)).toEqual({ ok: true, response }); + expect(f.cleanupCheckpoint).toHaveBeenCalledTimes(2); + expect(f.releaseExecution).toHaveBeenCalledTimes(2); + })); + +it('does not let a stale progress observation replace an enrolled generation or expose its output', () => + scope(async () => { + const f = fixture(); + await f.service.create(envelope(), 'same'); + const record = await admit([...f.deliveries.values()][0]); + f.deps.observe = async () => ({ + response: { ...completed(record), status: 'in_progress' }, + generationCreatedAt: 20, + }); + expect(await f.service.retrieve(owner, record.responseId)).toMatchObject({ + ok: true, + response: { output: [] }, + }); + expect((await storage.read(owner, record.responseId))?.generationCreatedAt).toBe(10); + })); + +it('rejects a runtime result carrying a different response identity', () => + scope(async () => { + const { service, deps, deliveries } = fixture(); + await service.create(envelope(), 'same'); + const record = [...deliveries.values()][0]; + deps.observe = async () => ({ response: { ...completed(record), id: 'resp_other' } }); + await expect(service.retrieve(owner, record.responseId)).rejects.toThrow( + 'execution identity changed', + ); + expect((await storage.read(owner, record.responseId))?.status).toBe('queued'); + })); + +it.each([{ store: false }, { stream: true }, { background: false }])( + 'rejects unsupported background combinations before publication: %p', + (payload) => + scope(async () => { + const { service, deliveries } = fixture(); + expect(await service.create(envelope(payload), 'same')).toEqual({ + ok: false, + status: 400, + code: 'invalid_background_request', + }); + expect(deliveries.size).toBe(0); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); + }), +); + +it.each(['queued', 'pending', 'completed'] as const)( + 'reauthorizes the original source key before replaying a %s record with a replacement key', + (state) => + scope(async () => { + const { deps, service, deliveries } = fixture(); + const allowed = new Set(['key-a', 'key-b']); + const authorize = jest.fn( + async (_owner: ResponseOwner, _agent: string, key?: string) => + key != null && allowed.has(key), + ); + deps.authorize = authorize; + expect((await service.create(envelope(), 'same-request', 'key-a')).ok).toBe(true); + let record = [...deliveries.values()][0]; + const response = completed(record); + if (state === 'completed') { + expect( + ( + await storage.publish(owner, record.responseId, record.revision, { + status: 'completed', + responseJSON: JSON.stringify(response), + }) + ).ok, + ).toBe(true); + } else if (state === 'pending') { + const running = await storage.publish(owner, record.responseId, record.revision, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: JSON.stringify({ ...response, status: 'in_progress' }), + }); + if (!running.ok) throw new Error('Publication failed'); + record = running.record; + expect( + ( + await storage.stageTerminal(owner, record.responseId, record.revision, { + generationCreatedAt: 10, + providerExecutionId: 'provider', + status: 'completed', + responseJSON: JSON.stringify(response), + }) + ).ok, + ).toBe(true); + } + deliveries.clear(); + deps.enqueue = jest.fn(); + deps.observe = jest.fn(); + deps.recoverTerminal = jest.fn(); + allowed.delete('key-a'); + authorize.mockClear(); + expect(await service.create(envelope(), 'same-request', 'key-b')).toEqual({ + ok: false, + status: 403, + code: 'permission_denied', + }); + expect(authorize.mock.calls.map((call) => call[2])).toEqual(['key-b', 'key-a']); + expect(deps.enqueue).not.toHaveBeenCalled(); + expect(deps.observe).not.toHaveBeenCalled(); + expect(deps.recoverTerminal).not.toHaveBeenCalled(); + expect(deliveries.size).toBe(0); + }), +); + +it('does not add a second source-authority lookup for a newly reserved response', () => + scope(async () => { + const { deps, service } = fixture(); + deps.authorize = jest.fn().mockResolvedValue(true); + expect((await service.create(envelope(), 'new-request', 'key')).ok).toBe(true); + expect(deps.authorize).toHaveBeenCalledTimes(1); + })); + +it('rejects JavaScript composition without a completion or execution-release owner before any reservation', async () => { + const f = fixture(); + expect(() => + Reflect.apply(createBackgroundResponseService, undefined, [ + { ...f.deps, recoverTerminal: undefined }, + ]), + ).toThrow('completion owner'); + expect(() => + Reflect.apply(createResponseCompletion, undefined, [ + { storage, cleanupCheckpoint: async () => {} }, + ]), + ).toThrow('cleanup and release owners'); + expect(await runAsSystem(() => mongoose.models.StoredResponse.countDocuments({}))).toBe(0); +}); diff --git a/packages/api/src/agents/responses/background.ts b/packages/api/src/agents/responses/background.ts new file mode 100644 index 0000000000..0619661706 --- /dev/null +++ b/packages/api/src/agents/responses/background.ts @@ -0,0 +1,213 @@ +import { createHash, randomUUID } from 'node:crypto'; +import type { ResponseOwner, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; +import type { ResponsesRunEnvelope, JsonValue } from '../envelope'; +import type { ResponseCompletionRecord } from './completion'; +import type { Response } from './types'; +import { + createResponseContext, + createResponseAggregator, + buildAggregatedResponse, +} from './service'; + +export type BackgroundResponseResult = + | { ok: true; response: Response } + | { ok: false; status: 400 | 403 | 404 | 409; code: string }; + +/** The host supplies the existing durable queue and Responses execution runtime. */ +export interface BackgroundResponseDependencies { + storage: ResponseStorage; + /** Recheck current credential/owner/Agent authority; possession of a response ID is not authority. */ + authorize: (owner: ResponseOwner, agentId: string, sourceKeyId?: string) => Promise; + resolveContinuation: ( + owner: ResponseOwner, + previousResponseId?: string, + ) => Promise<{ + conversationId: string; + parentMessageId: string; + expiredAt?: Date; + isTemporary?: boolean; + }>; + /** Idempotent queue publication under record.deliveryKey; MUST NOT execute inline or mint a new identity on retry. */ + enqueue: (record: StoredResponseRecord) => Promise; + deliveryKey?: (record: StoredResponseRecord) => string; + /** Progress-only projection. A terminal observation is a hint to reread the journal; + * generationCreatedAt is a consistency witness, never enrollment authority. */ + observe: (record: StoredResponseRecord) => Promise<{ + response: Response; + generationCreatedAt?: number; + } | null>; + /** Completion journal owns cleanup and release before any terminal view becomes public. */ + recoverTerminal: (record: StoredResponseRecord) => Promise; + /** Signal exact execution; the completion journal confirms its final outcome. Never claim a missing job stopped. */ + cancel: (record: StoredResponseRecord) => Promise; + now?: () => number; +} + +export function canonicalResponseJSON(request: object): string { + return JSON.stringify(request, (_key: string, value: JsonValue) => { + if (value === null || typeof value !== 'object' || Array.isArray(value)) return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, value[key]]), + ); + }); +} + +function terminal(status: StoredResponseRecord['status']) { + return ['completed', 'failed', 'incomplete', 'cancelled'].includes(status); +} + +export function buildPendingBackgroundResponse( + record: StoredResponseRecord, + envelope?: ResponsesRunEnvelope, +): Response { + const request = + envelope?.payload ?? (JSON.parse(record.requestJSON) as ResponsesRunEnvelope['payload']); + const context = createResponseContext(request, record.responseId); + context.createdAt = Math.floor(record.createdAt.getTime() / 1000); + return { + ...buildAggregatedResponse(context, createResponseAggregator()), + status: 'queued', + completed_at: null, + output: [], + usage: null, + background: true, + store: true, + metadata: request.metadata ?? {}, + }; +} + +function view(record: StoredResponseRecord): Response { + return record.responseJSON + ? (JSON.parse(record.responseJSON) as Response) + : buildPendingBackgroundResponse(record); +} + +/** Provider-neutral Responses lifecycle; no channel, Slack, service-token or rendering dependency. */ +export function createBackgroundResponseService(deps: BackgroundResponseDependencies) { + if (typeof deps.recoverTerminal !== 'function') + throw new Error('Response completion owner is unavailable'); + const now = deps.now ?? Date.now; + + async function readAuthorized(owner: ResponseOwner, responseId: string) { + const record = await deps.storage.read(owner, responseId); + if (!record || !(await deps.authorize(owner, record.agentId, record.sourceKeyId))) return null; + return record; + } + + function hasTerminalWork(record: StoredResponseRecord): boolean { + return record.pendingTerminal != null || terminal(record.status); + } + + async function canonicalView(record: StoredResponseRecord): Promise { + if (!hasTerminalWork(record)) return view(record); + const recovered = await deps.recoverTerminal(record); + return recovered.retirement ? null : view(recovered); + } + + async function freshView(record: StoredResponseRecord): Promise { + const current = await deps.storage.read(record, record.responseId); + return current ? canonicalView(current) : null; + } + + async function refresh(record: StoredResponseRecord): Promise { + if (hasTerminalWork(record)) return canonicalView(record); + const observed = await deps.observe(record); + if (!observed) return view(record); + const response = observed.response; + if ( + response.id !== record.responseId || + response.model !== record.agentId || + !response.background || + !response.store + ) + throw new Error('Background response execution identity changed'); + // An observer has no provider-drain or checkpoint proof. Only the journal may finalize it. + // Enrollment identity likewise belongs to admission, never a progress projection. + if ( + terminal(response.status) || + (observed.generationCreatedAt != null && + observed.generationCreatedAt !== record.generationCreatedAt) + ) + return freshView(record); + const result = await deps.storage.publish(record, record.responseId, record.revision, { + status: + response.required_action || (record.status === 'requires_action' && record.review != null) + ? 'requires_action' + : response.status, + responseJSON: JSON.stringify(response), + }); + return result.ok ? canonicalView(result.record) : freshView(record); + } + + async function responseResult(record: StoredResponseRecord): Promise { + const response = await refresh(record); + return response + ? { ok: true, response } + : { ok: false, status: 404, code: 'response_not_found' }; + } + + return { + async create( + envelope: ResponsesRunEnvelope, + idempotencyKey?: string, + sourceKeyId?: string, + ): Promise { + const request = envelope.payload; + if (request.background !== true || request.store === false || request.stream === true) + return { ok: false, status: 400, code: 'invalid_background_request' }; + if (idempotencyKey != null && (!idempotencyKey.trim() || idempotencyKey.length > 256)) + return { ok: false, status: 400, code: 'invalid_idempotency_key' }; + const owner = envelope.principal; + if (!(await deps.authorize(owner, request.model, sourceKeyId))) + return { ok: false, status: 403, code: 'permission_denied' }; + const normalized = { ...request, store: true }; + const requestJSON = canonicalResponseJSON(normalized); + const continuation = await deps.resolveContinuation(owner, request.previous_response_id); + const result = await deps.storage.reserve({ + ...owner, + ...continuation, + agentId: request.model, + ...(sourceKeyId == null ? {} : { sourceKeyId }), + responseId: `resp_${randomUUID().replaceAll('-', '')}`, + requestKey: idempotencyKey ?? randomUUID(), + fingerprint: createHash('sha256').update(requestJSON).digest('hex'), + requestJSON, + }); + if (!result.ok) return { ok: false, status: 409, code: result.code }; + let record = result.record; + if (result.replayed && !(await deps.authorize(owner, record.agentId, record.sourceKeyId))) + return { ok: false, status: 403, code: 'permission_denied' }; + if (hasTerminalWork(record)) return responseResult(record); + const attached = await deps.storage.attachDelivery( + owner, + record.responseId, + deps.deliveryKey?.(record) ?? `response_${record.responseId}`, + ); + if (attached.ok) { + record = attached.record; + if (!hasTerminalWork(record)) await deps.enqueue(record); + } + return hasTerminalWork(record) + ? responseResult(record) + : { ok: true, response: view(record) }; + }, + async retrieve(owner: ResponseOwner, responseId: string): Promise { + const record = await readAuthorized(owner, responseId); + if (!record) return { ok: false, status: 404, code: 'response_not_found' }; + return responseResult(record); + }, + async cancel(owner: ResponseOwner, responseId: string): Promise { + const authorized = await readAuthorized(owner, responseId); + if (!authorized) return { ok: false, status: 404, code: 'response_not_found' }; + if (hasTerminalWork(authorized)) return responseResult(authorized); + const record = await deps.storage.requestCancellation(owner, responseId, now()); + if (!record) return { ok: false, status: 404, code: 'response_not_found' }; + if (!hasTerminalWork(record)) await deps.cancel(record); + const current = await readAuthorized(owner, responseId); + if (!current) return { ok: false, status: 404, code: 'response_not_found' }; + return responseResult(current); + }, + }; +} diff --git a/packages/api/src/agents/responses/checkpoint.ts b/packages/api/src/agents/responses/checkpoint.ts new file mode 100644 index 0000000000..97aa067a61 --- /dev/null +++ b/packages/api/src/agents/responses/checkpoint.ts @@ -0,0 +1,49 @@ +import { checkpointerSchema } from 'librechat-data-provider'; +import type { StoredResponseRecord } from '@librechat/data-schemas'; +import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints'; +import { deleteAgentCheckpoint } from '../checkpointer'; + +/** Cleanup uses captured server settings, never current config or caller-supplied collection names. */ +export type ResponseCheckpointIdentity = Pick< + StoredResponseRecord, + 'userId' | 'tenantId' | 'conversationId' +> & { + review?: Pick, 'checkpointConfigJSON'>; + pendingTerminal?: Pick< + NonNullable, + 'checkpointNamespace' | 'checkpointConfigJSON' + >; +}; + +export async function cleanupResponseCheckpoint( + record: ResponseCheckpointIdentity, + checkpointNamespace: string, +): Promise { + const receipt = record.pendingTerminal; + const raw = receipt?.checkpointConfigJSON ?? record.review?.checkpointConfigJSON; + if (!raw || !checkpointNamespace) + throw new Error('Response checkpoint cleanup identity unavailable'); + if ( + (receipt?.checkpointNamespace && receipt.checkpointNamespace !== checkpointNamespace) || + (receipt?.checkpointConfigJSON && + record.review?.checkpointConfigJSON && + receipt.checkpointConfigJSON !== record.review.checkpointConfigJSON) + ) + throw new Error('Response checkpoint cleanup identity changed'); + if ( + !checkpointNamespace.startsWith(checkpointOwnerNamespacePrefix(record.userId, record.tenantId)) + ) + throw new Error('Response checkpoint owner changed'); + const config = checkpointerSchema.parse(JSON.parse(raw)); + if ( + !config || + config.type !== 'mongo' || + !config.checkpointCollectionName || + !config.checkpointWritesCollectionName + ) + throw new Error('Response checkpoint must retain its durable storage settings'); + await deleteAgentCheckpoint(record.conversationId, config, undefined, { + throwOnError: true, + checkpointNamespace, + }); +} diff --git a/packages/api/src/agents/responses/completion.spec.ts b/packages/api/src/agents/responses/completion.spec.ts new file mode 100644 index 0000000000..ad2559cca8 --- /dev/null +++ b/packages/api/src/agents/responses/completion.spec.ts @@ -0,0 +1,141 @@ +import type { ToolCompleteEvent } from '@librechat/agents'; +import type { Response as HttpResponse } from 'express'; +import type { Response } from './types'; +import { + buildAggregatedResponse, + createResponseAggregator, + createResponsesEventHandlers, + createAggregatorEventHandlers, +} from './service'; +import { createResponseTracker } from './handlers'; + +const context = { responseId: 'resp_completion', model: 'agent', createdAt: 1 }; +function setup(stream: boolean, announceArguments = true) { + const events: Array<{ type: string; response?: Response; call_id?: string; arguments?: string }> = + []; + const res = { + write: (chunk: string) => { + if (chunk.startsWith('data: ') && !chunk.includes('[DONE]')) + events.push(JSON.parse(chunk.slice(6))); + }, + } as HttpResponse; + const streaming = createResponsesEventHandlers({ + res, + tracker: createResponseTracker(), + context, + }); + const aggregator = createResponseAggregator(); + const handlers = stream ? streaming.handlers : createAggregatorEventHandlers(aggregator); + handlers.on_run_step.handle('on_run_step', { + id: 'step-a', + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'call-a', name: 'protected_lookup' }], + }, + }); + if (announceArguments) + handlers.on_chat_model_end.handle('on_chat_model_end', { + output: { tool_calls: [{ id: 'call-a', args: {} }] }, + }); + return { + handlers, + events, + finish: () => { + if (!stream) return buildAggregatedResponse(context, aggregator); + streaming.finalizeStream(); + return events.find((event) => event.type === 'response.completed')!.response!; + }, + }; +} +function completed(output: string, id = 'call-a') { + return { + result: { + id: 'step-a', + index: 0, + type: 'tool_call', + tool_call: { id, name: 'protected_lookup', args: '{}', output, progress: 1 }, + }, + } satisfies { result: ToolCompleteEvent }; +} + +describe.each([false, true])('authoritative SDK tool completion (stream=%s)', (stream) => { + it('finishes eager calls with exact arguments and ignores fragments that arrive after completion', () => { + const f = setup(stream, false); + f.handlers.on_run_step_delta.handle('on_run_step_delta', { + id: 'step-a', + delta: { + type: 'tool_calls', + tool_calls: [{ id: 'call-a', args: '{"org":' }], + }, + }); + const event = completed('actual-result'); + event.result.tool_call.args = '{"org":"a"}'; + f.handlers.on_run_step_completed.handle('on_run_step_completed', event); + const countAfterCompletion = f.events.length; + f.handlers.on_run_step_delta.handle('on_run_step_delta', { + id: 'step-a', + delta: { + type: 'tool_calls', + tool_calls: [{ id: 'call-a', args: '"a"}' }], + }, + }); + f.handlers.on_chat_model_end.handle('on_chat_model_end', { + output: { tool_calls: [{ id: 'call-a', args: { org: 'a' } }] }, + }); + expect(f.events).toHaveLength(countAfterCompletion); + expect(f.finish().output.find((item) => item.type === 'function_call')).toMatchObject({ + arguments: '{"org":"a"}', + status: 'completed', + }); + if (stream) + expect( + f.events.find((entry) => entry.type === 'response.function_call_arguments.done'), + ).toMatchObject({ call_id: 'call-a', arguments: '{"org":"a"}' }); + }); + + it.each(['{"result":[1,2]}', ''])( + 'retains the actual result including empty output: %s', + (output) => { + const f = setup(stream); + const event = completed(output); + f.handlers.on_run_step_completed.handle('on_run_step_completed', event); + f.handlers.on_run_step_completed.handle('on_run_step_completed', event); + const response = f.finish(); + expect(response.output.filter((item) => item.type === 'function_call_output')).toEqual([ + expect.objectContaining({ type: 'function_call_output', call_id: 'call-a', output }), + ]); + expect(response.output.find((item) => item.type === 'function_call')).toMatchObject({ + status: 'completed', + arguments: '{}', + }); + }, + ); + + it('does not copy unrelated completion, summary or raw artifacts into the response', () => { + const f = setup(stream); + f.handlers.on_run_step_completed.handle( + 'on_run_step_completed', + completed('unrelated-secret', 'unknown-call'), + ); + f.handlers.on_run_step_completed.handle('on_run_step_completed', { + result: { type: 'summary', summary: 'private-summary' }, + }); + f.handlers.on_run_step_completed.handle('on_run_step_completed', { + result: { tool_call: { id: 'call-a', output: { artifact: { token: 'private-token' } } } }, + }); + expect(f.finish().output.some((item) => item.type === 'function_call_output')).toBe(false); + expect(JSON.stringify(f.events)).not.toMatch(/unrelated-secret|private-summary|private-token/); + }); + + it('keeps compatibility with legacy flat completions without emitting a second result', () => { + const f = setup(stream); + f.handlers.on_tool_end.handle('on_tool_end', { + tool_call_id: 'call-a', + output: 'first-result', + }); + f.handlers.on_run_step_completed.handle('on_run_step_completed', completed('first-result')); + expect(f.finish().output.filter((item) => item.type === 'function_call_output')).toEqual([ + expect.objectContaining({ call_id: 'call-a', output: 'first-result' }), + ]); + }); +}); diff --git a/packages/api/src/agents/responses/completion.ts b/packages/api/src/agents/responses/completion.ts index 25177f066f..25d21f044b 100644 --- a/packages/api/src/agents/responses/completion.ts +++ b/packages/api/src/agents/responses/completion.ts @@ -1,198 +1,296 @@ -import { EModelEndpoint } from 'librechat-data-provider'; -import type { AppConfig } from '@librechat/data-schemas'; -import type { Response as HttpResponse } from 'express'; -import type { TurnConversationRequest, ConversationWriteContext } from '~/conversations/save'; -import type { PersistedReply, ReplyAnnouncement } from '~/conversations/announce'; -import type { getLangfuseTraceMessageFields } from '~/langfuse/destinations'; -import type { ConversationCodeEnvironmentDecision } from '~/code/decision'; -import type { Response, Usage, ResponseAccounting } from './types'; -import type { ForegroundResponseTerminal } from './foreground'; -import type { InternalMessage } from './service'; -import { - saveForegroundResponseInputs, - projectStoredResponseMessage, - requireResponseStorageWrite, -} from './foreground'; -import { resolvePersistableCodeEnvironmentDecision } from '~/code/decision'; -import { getConversationWriteContext } from '~/conversations/save'; -import { isTemporaryTurn } from '~/conversations/retention'; -import { sendResponsesErrorResponse } from './service'; +import { isDeepStrictEqual } from 'node:util'; +import type { + ResponseStorage, + StoredResponseRecord, + ResponseTerminalReceipt, + ResponseMutationResult, + ResponseCleanupIdentity, +} from '@librechat/data-schemas'; +import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { responseTenantId } from './scope'; -interface CompletionRequest - extends Omit { - config?: AppConfig; - resolvedConversation?: - | (ReturnType & { - conversationId?: string; - isTemporary?: boolean; - expiredAt?: Date; - }) - | null; +/** A retirement result carries cleanup authority only and must never render private output. */ +export interface ResponseCompletionRecord extends StoredResponseRecord { + retirement?: ResponseCleanupIdentity['retirement']; } -type InputMessage = Parameters< - Parameters>[1]['saveMessage'] ->[1]; -type OutputMessage = ReturnType & { - messageId: string; - conversationId: string; - isCreatedByUser: false; - sender: string; - endpoint: string; - model: string; - finish_reason: string; - tokenCount?: number; + +export interface ResponseCompletion { + finish(record: StoredResponseRecord): Promise; + stageAndFinish( + record: StoredResponseRecord, + receipt: ResponseTerminalReceipt, + ): Promise; +} + +export interface ResponseCompletionDependencies { + storage: ResponseStorage; + cleanupCheckpoint: (record: StoredResponseRecord, namespace: string) => Promise; + /** Required and idempotent: transient provider ownership survives until canonical publication. */ + releaseExecution: ( + record: StoredResponseRecord, + receipt?: ResponseTerminalReceipt, + ) => Promise; +} + +function terminal(record: StoredResponseRecord): boolean { + return ['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status); +} + +type ExecutionReleaseIdentity = Pick< + StoredResponseRecord, + | 'userId' + | 'tenantId' + | 'responseId' + | 'agentId' + | 'conversationId' + | 'generationCreatedAt' + | 'expiredAt' + | 'status' +> & { + retirement?: ResponseCleanupIdentity['retirement']; + resume?: Pick, 'state' | 'providerExecutionId'>; + review?: Pick, 'providerExecutionId'>; }; -interface StoredReply extends PersistedReply { - conversationId: string; - messageId: string; -} -interface CompletionDependencies { - saveMessage: ( - context: ConversationWriteContext, - message: InputMessage | OutputMessage, - options: { context: string }, - ) => Promise; - saveConvo: ( - context: ConversationWriteContext, - fields: ReturnType & { - conversationId: string; - endpoint: string; - agent_id: string; - title?: string; - model?: string; - }, - options: { context: string; initialAgentId: string | null }, - ) => Promise; - announceReply: (announcement: ReplyAnnouncement) => Promise; - resolveTitle: (request: CompletionRequest, candidate: string) => string | null; - traceFields: typeof getLangfuseTraceMessageFields; + +/** Release transient provider proof only after its canonical terminal result is durable. */ +export function createResponseExecutionRelease( + jobs: Pick, +): ( + record: ExecutionReleaseIdentity, + receipt?: Pick, +) => Promise { + return async (record, receipt) => { + if ( + !['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status) && + !record.retirement && + !(record.expiredAt && record.expiredAt.getTime() <= Date.now()) + ) + throw new Error('Response terminal publication is unavailable'); + if (record.generationCreatedAt == null) return; + const job = await jobs.getJob(record.responseId); + if (!job) return; + const expectedProvider = + receipt?.providerExecutionId ?? + (record.resume?.state === 'admitted' + ? record.resume.providerExecutionId + : record.review?.providerExecutionId); + if ( + job.createdAt !== record.generationCreatedAt || + job.metadata.userId !== record.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) || + job.metadata.agent_id !== record.agentId || + job.metadata.endpoint !== 'responses' || + job.metadata.responseMessageId !== record.responseId || + job.metadata.conversationId !== record.conversationId || + !['complete', 'error', 'aborted'].includes(job.status) || + job.metadata.providerDrained !== true || + !job.metadata.providerExecutionId || + (expectedProvider != null && job.metadata.providerExecutionId !== expectedProvider) || + (receipt != null && job.metadata.providerExecutionOwnerId !== receipt.ownerId) + ) + throw new Error('Response terminal execution identity changed'); + const released = await jobs.finishProviderTerminalPersistence( + record.responseId, + record.generationCreatedAt, + job.metadata.providerExecutionId, + receipt?.ownerId ?? job.metadata.providerExecutionOwnerId, + ); + if (!released) throw new Error('Response terminal execution release is unavailable'); + }; } -/** Own storage ordering and transport selection in the same typed completion boundary. */ -export function stageForegroundResponse( - { - terminal, - request, - response, - conversationId, - agentId, - agent, - codeEnvironmentDecision, - messages, - parentMessageId, - visibleOutputTokens, - transport, - }: { - terminal: ForegroundResponseTerminal; - request: CompletionRequest; - response: Response; - conversationId: string; - agentId: string; - agent: { id?: string; name?: string; model?: string }; - codeEnvironmentDecision?: ConversationCodeEnvironmentDecision; - messages: (InternalMessage & { messageId: string; parentMessageId: string })[]; - parentMessageId: string; - visibleOutputTokens?: number; - transport: - | { type: 'json'; response: HttpResponse } - | { - type: 'stream'; - response: HttpResponse; - finalize: (usage?: Usage, accounting?: ResponseAccounting, response?: Response) => void; - fail: Parameters['stage']>[0]['fail']; - }; - }, - deps: CompletionDependencies, -): void { - terminal.stage({ - response, - save: async (result) => { - const context = getConversationWriteContext(request); - const title = deps.resolveTitle(request, agent.name || 'Open Responses Conversation'); - await requireResponseStorageWrite( - deps.saveConvo( - context, - { - conversationId, - endpoint: EModelEndpoint.agents, - agent_id: agentId, - ...resolvePersistableCodeEnvironmentDecision({ - conversationId, - decision: codeEnvironmentDecision, - conversation: - request.resolvedConversation?.conversationId != null - ? { - conversationId: request.resolvedConversation.conversationId, - codeEnvironmentMode: request.resolvedConversation.codeEnvironmentMode, - codeWorkspaces: request.resolvedConversation.codeWorkspaces, - } - : undefined, - }), - ...(title != null && { title }), - model: agent.model, - }, - { - context: 'Responses API - save conversation', - initialAgentId: agent.id === agentId ? agentId : null, - }, - ), - { conversationId }, - ); - await saveForegroundResponseInputs( - { context, conversationId, messages, agentId }, - { saveMessage: deps.saveMessage }, - ); - const traceFields = await deps.traceFields(request.config, result.id, { - isTemporary: isTemporaryTurn(request), - }); - return requireResponseStorageWrite( - deps.saveMessage( - context, - { - messageId: result.id, - conversationId, - ...projectStoredResponseMessage(result, parentMessageId), - isCreatedByUser: false, - ...traceFields, - sender: 'Agent', - endpoint: EModelEndpoint.agents, - model: agentId, - finish_reason: result.status === 'completed' ? 'stop' : result.status, - tokenCount: visibleOutputTokens ?? result.usage?.output_tokens, - }, - { context: 'Responses API - save assistant response' }, - ), - { conversationId, messageId: result.id }, - ).then((saved) => { - if (!saved) throw new Error('Response write returned no reply'); - return saved; - }); - }, - afterStore: async (reply) => { - await deps.announceReply({ - userId: request.user?.id, - conversationId, - reply, - context: 'Responses API - announce stored reply', - }); - }, - fail: (failure) => { - if (transport.type === 'stream') return transport.fail(failure); - sendResponsesErrorResponse( - transport.response, - failure.status, - failure.message, - 'server_error', - failure.code, - ); - }, - publish: (result) => { - if (transport.type === 'json') { - transport.response.json(result); - return; - } - transport.finalize(result.usage ?? undefined, result.accounting, result); - transport.response.end(); - }, - }); +function assertReceipt(record: StoredResponseRecord, receipt: ResponseTerminalReceipt): void { + let response: unknown; + try { + response = JSON.parse(receipt.responseJSON); + } catch { + throw new Error('Response terminal receipt is invalid'); + } + if ( + receipt.generationCreatedAt !== record.generationCreatedAt || + !response || + typeof response !== 'object' || + !('id' in response) || + response.id !== record.responseId || + !('model' in response) || + response.model !== record.agentId || + !('status' in response) || + response.status !== receipt.status || + !('object' in response) || + response.object !== 'response' || + !('output' in response) || + !Array.isArray(response.output) || + (receipt.checkpointConfigJSON != null && !receipt.checkpointNamespace) || + !('background' in response) || + response.background !== true || + !('store' in response) || + response.store !== true + ) + throw new Error('Response terminal receipt identity changed'); +} + +function sameExecution( + previous: StoredResponseRecord, + current: StoredResponseRecord, + receipt: ResponseTerminalReceipt, +): boolean { + const provider = + current.resume?.state === 'admitted' + ? current.resume.providerExecutionId + : current.review?.providerExecutionId; + return ( + current.userId === previous.userId && + responseTenantId(current.tenantId) === responseTenantId(previous.tenantId) && + current.responseId === previous.responseId && + current.agentId === previous.agentId && + current.sourceKeyId === previous.sourceKeyId && + current.conversationId === previous.conversationId && + current.generationCreatedAt === receipt.generationCreatedAt && + (provider == null || provider === receipt.providerExecutionId) && + isDeepStrictEqual(current.review, previous.review) && + isDeepStrictEqual(current.resume, previous.resume) + ); +} + +/** Persist a post-drain result before cleanup, then retry only cleanup/publication, never execution. */ +export function createResponseCompletion(deps: ResponseCompletionDependencies): ResponseCompletion { + if (typeof deps.cleanupCheckpoint !== 'function' || typeof deps.releaseExecution !== 'function') + throw new Error('Response completion cleanup and release owners are unavailable'); + async function discardRetired(record: StoredResponseRecord, receipt: ResponseTerminalReceipt) { + const identity = await deps.storage.readCleanupIdentity(record, record.responseId, { + ...record, + checkpointNamespace: receipt.checkpointNamespace, + checkpointConfigJSON: receipt.checkpointConfigJSON, + }); + if ( + !identity?.retirement && + !(identity?.expiredAt && identity.expiredAt.getTime() <= Date.now()) + ) + return null; + const retired: ResponseCompletionRecord = { + ...record, + ...identity, + retirement: identity.retirement ?? 'expired', + }; + const namespace = identity.checkpointNamespace ?? receipt.checkpointNamespace; + if (identity.requiresCheckpointCleanup && !namespace) + throw new Error('Response checkpoint cleanup identity unavailable'); + if (namespace) + await deps.cleanupCheckpoint( + { + ...retired, + review: undefined, + pendingTerminal: { + ...receipt, + checkpointNamespace: namespace, + checkpointConfigJSON: identity.checkpointConfigJSON ?? receipt.checkpointConfigJSON, + }, + }, + namespace, + ); + await deps.releaseExecution(retired, receipt); + return { + ...retired, + requestJSON: '', + responseJSON: undefined, + review: undefined, + resume: undefined, + pendingTerminal: undefined, + status: receipt.status, + }; + } + async function finish(record: StoredResponseRecord): Promise { + const receipt = record.pendingTerminal; + if (!receipt) { + if (terminal(record)) await deps.releaseExecution(record); + return record; + } + assertReceipt(record, receipt); + if (receipt.checkpointNamespace) + await deps.cleanupCheckpoint(record, receipt.checkpointNamespace); + let current: StoredResponseRecord | null; + try { + const result = await deps.storage.finishTerminal( + record, + record.responseId, + record.revision, + receipt, + ); + current = result.ok ? result.record : await deps.storage.read(record, record.responseId); + } catch (error) { + current = await deps.storage.read(record, record.responseId); + if (!current) { + const expired = await discardRetired(record, receipt); + if (expired) return expired; + } + if ( + !current || + current.status !== receipt.status || + current.generationCreatedAt !== receipt.generationCreatedAt || + current.responseJSON !== receipt.responseJSON || + current.pendingTerminal + ) + throw error; + } + if (!current) { + const expired = await discardRetired(record, receipt); + if (expired) return expired; + throw new Error('Response terminal journal is unavailable'); + } + if (terminal(current)) await deps.releaseExecution(current, receipt); + return current; + } + return { + finish, + async stageAndFinish( + record: StoredResponseRecord, + receipt: ResponseTerminalReceipt, + ): Promise { + assertReceipt(record, receipt); + if (record.expiredAt && record.expiredAt.getTime() <= Date.now()) { + const retired = await discardRetired(record, receipt); + if (!retired) throw new Error('Response retirement identity unavailable'); + return retired; + } + let staged: ResponseMutationResult; + try { + staged = await deps.storage.stageTerminal( + record, + record.responseId, + { execution: record }, + receipt, + ); + } catch (error) { + const current = await deps.storage.read(record, record.responseId); + if (!current) { + const expired = await discardRetired(record, receipt); + if (expired) return expired; + } + const saved = current?.pendingTerminal; + if ( + !current || + !saved || + saved.generationCreatedAt !== receipt.generationCreatedAt || + saved.providerExecutionId !== receipt.providerExecutionId || + saved.ownerId !== receipt.ownerId || + saved.responseJSON !== receipt.responseJSON || + saved.status !== receipt.status || + saved.checkpointNamespace !== receipt.checkpointNamespace || + saved.checkpointConfigJSON !== receipt.checkpointConfigJSON + ) + throw error; + return finish(current); + } + if (staged.ok) return finish(staged.record); + const current = await deps.storage.read(record, record.responseId); + if (!current) { + const expired = await discardRetired(record, receipt); + if (expired) return expired; + } + if (!current || !sameExecution(record, current, receipt)) + throw new Error('Response terminal receipt publication conflicted'); + if (terminal(current)) return finish(current); + throw new Error('Response terminal receipt publication conflicted'); + }, + }; } diff --git a/packages/api/src/agents/responses/foreground.ts b/packages/api/src/agents/responses/foreground.ts index 26f3be9a6a..b639b898e5 100644 --- a/packages/api/src/agents/responses/foreground.ts +++ b/packages/api/src/agents/responses/foreground.ts @@ -1,14 +1,23 @@ import { ContentTypes, EModelEndpoint } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { Response as HttpResponse } from 'express'; import type { RequestHandler } from 'express'; +import type { TurnConversationRequest, ConversationWriteContext } from '~/conversations/save'; +import type { Response, ResponseContext, Usage, ResponseAccounting } from './types'; +import type { PersistedReply, ReplyAnnouncement } from '~/conversations/announce'; +import type { getLangfuseTraceMessageFields } from '~/langfuse/destinations'; +import type { ConversationCodeEnvironmentDecision } from '~/code/decision'; import type { AgentExecutionEnrollment } from '../remote/lifecycle'; import type { StoredResponseMessage } from './persistence'; -import type { Response, ResponseContext } from './types'; import type { InternalMessage } from './service'; import { ResponsePersistenceError, selectResponseHistory, projectResponseMessage, } from './persistence'; +import { resolvePersistableCodeEnvironmentDecision } from '~/code/decision'; +import { getConversationWriteContext } from '~/conversations/save'; +import { isTemporaryTurn } from '~/conversations/retention'; import { sendResponsesErrorResponse } from './service'; import { getSafeErrorMetadata } from '~/utils/errors'; @@ -347,3 +356,182 @@ export function createForegroundResponseTerminal({ }, }; } + +interface CompletionRequest + extends Omit { + config?: AppConfig; + resolvedConversation?: + | (ReturnType & { + conversationId?: string; + isTemporary?: boolean; + expiredAt?: Date; + }) + | null; +} +type InputMessage = Parameters< + Parameters>[1]['saveMessage'] +>[1]; +type OutputMessage = ReturnType & { + messageId: string; + conversationId: string; + isCreatedByUser: false; + sender: string; + endpoint: string; + model: string; + finish_reason: string; + tokenCount?: number; +}; +interface StoredReply extends PersistedReply { + conversationId: string; + messageId: string; +} +interface CompletionDependencies { + saveMessage: ( + context: ConversationWriteContext, + message: InputMessage | OutputMessage, + options: { context: string }, + ) => Promise; + saveConvo: ( + context: ConversationWriteContext, + fields: ReturnType & { + conversationId: string; + endpoint: string; + agent_id: string; + title?: string; + model?: string; + }, + options: { context: string; initialAgentId: string | null }, + ) => Promise; + announceReply: (announcement: ReplyAnnouncement) => Promise; + resolveTitle: (request: CompletionRequest, candidate: string) => string | null; + traceFields: typeof getLangfuseTraceMessageFields; +} + +/** Own storage ordering and transport selection in the same typed completion boundary. */ +export function stageForegroundResponse( + { + terminal, + request, + response, + conversationId, + agentId, + agent, + codeEnvironmentDecision, + messages, + parentMessageId, + visibleOutputTokens, + transport, + }: { + terminal: ForegroundResponseTerminal; + request: CompletionRequest; + response: Response; + conversationId: string; + agentId: string; + agent: { id?: string; name?: string; model?: string }; + codeEnvironmentDecision?: ConversationCodeEnvironmentDecision; + messages: (InternalMessage & { messageId: string; parentMessageId: string })[]; + parentMessageId: string; + visibleOutputTokens?: number; + transport: + | { type: 'json'; response: HttpResponse } + | { + type: 'stream'; + response: HttpResponse; + finalize: (usage?: Usage, accounting?: ResponseAccounting, response?: Response) => void; + fail: Parameters['stage']>[0]['fail']; + }; + }, + deps: CompletionDependencies, +): void { + terminal.stage({ + response, + save: async (result) => { + const context = getConversationWriteContext(request); + const title = deps.resolveTitle(request, agent.name || 'Open Responses Conversation'); + await requireResponseStorageWrite( + deps.saveConvo( + context, + { + conversationId, + endpoint: EModelEndpoint.agents, + agent_id: agentId, + ...resolvePersistableCodeEnvironmentDecision({ + conversationId, + decision: codeEnvironmentDecision, + conversation: + request.resolvedConversation?.conversationId != null + ? { + conversationId: request.resolvedConversation.conversationId, + codeEnvironmentMode: request.resolvedConversation.codeEnvironmentMode, + codeWorkspaces: request.resolvedConversation.codeWorkspaces, + } + : undefined, + }), + ...(title != null && { title }), + model: agent.model, + }, + { + context: 'Responses API - save conversation', + initialAgentId: agent.id === agentId ? agentId : null, + }, + ), + { conversationId }, + ); + await saveForegroundResponseInputs( + { context, conversationId, messages, agentId }, + { saveMessage: deps.saveMessage }, + ); + const traceFields = await deps.traceFields(request.config, result.id, { + isTemporary: isTemporaryTurn(request), + }); + return requireResponseStorageWrite( + deps.saveMessage( + context, + { + messageId: result.id, + conversationId, + ...projectStoredResponseMessage(result, parentMessageId), + isCreatedByUser: false, + ...traceFields, + sender: 'Agent', + endpoint: EModelEndpoint.agents, + model: agentId, + finish_reason: result.status === 'completed' ? 'stop' : result.status, + tokenCount: visibleOutputTokens ?? result.usage?.output_tokens, + }, + { context: 'Responses API - save assistant response' }, + ), + { conversationId, messageId: result.id }, + ).then((saved) => { + if (!saved) throw new Error('Response write returned no reply'); + return saved; + }); + }, + afterStore: async (reply) => { + await deps.announceReply({ + userId: request.user?.id, + conversationId, + reply, + context: 'Responses API - announce stored reply', + }); + }, + fail: (failure) => { + if (transport.type === 'stream') return transport.fail(failure); + sendResponsesErrorResponse( + transport.response, + failure.status, + failure.message, + 'server_error', + failure.code, + ); + }, + publish: (result) => { + if (transport.type === 'json') { + transport.response.json(result); + return; + } + transport.finalize(result.usage ?? undefined, result.accounting, result); + transport.response.end(); + }, + }); +} diff --git a/packages/api/src/agents/responses/index.ts b/packages/api/src/agents/responses/index.ts index e6ad4ebb91..a522e38fcd 100644 --- a/packages/api/src/agents/responses/index.ts +++ b/packages/api/src/agents/responses/index.ts @@ -13,6 +13,17 @@ export { projectResponseMessage, resolveResponseContinuation, } from './persistence'; +export { createBackgroundResponseService } from './background'; + +export { createBackgroundResponseAuthority } from './authority'; + +export { createBackgroundResponseQueue, backgroundResponseDeliveryKey } from './queue'; +export { + createBackgroundResponseWorkerAuth, + mintBackgroundResponseWorkerToken, + mintResponseResumeWorkerToken, +} from './worker'; +export type { BackgroundResponseDependencies, BackgroundResponseResult } from './background'; export { createResponseAccountingCollector, @@ -214,5 +225,4 @@ export { createClientToolHandoff, type ClientToolHandoff } from './clientTools'; export * from './foreground'; export * from './stored'; - -export * from './completion'; +export type { ResponseCompletion, ResponseCompletionRecord } from './completion'; diff --git a/packages/api/src/agents/responses/queue.spec.ts b/packages/api/src/agents/responses/queue.spec.ts new file mode 100644 index 0000000000..0f74d47ff6 --- /dev/null +++ b/packages/api/src/agents/responses/queue.spec.ts @@ -0,0 +1,449 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createResponseStorage, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { ResponseStorage, StoredResponseRecord, ResponseOwner } from '@librechat/data-schemas'; +import type { AgentTriggerService, AgentTriggerDeliveryPersistence } from '../triggers/service'; +import { + createBackgroundResponseQueue, + createBackgroundResponseEnvelope, + createResponseResumeQueue, + createResponseResumeEnvelope, + BACKGROUND_RESPONSE_RESUME_SOURCE, +} from './queue'; +import { createAgentTriggerService } from '../triggers/service'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +let methods: ReturnType; +let service: AgentTriggerService | undefined; +const owner = { userId: '000000000000000000000001', tenantId: 'response-queue-test' }; +const scope = (work: () => Promise) => tenantStorage.run({ tenantId: owner.tenantId }, work); + +function persistence(): AgentTriggerDeliveryPersistence { + return { + ...methods, + getAgentTriggerDeliveryBatch: (delivery) => + methods.getAgentTriggerDeliveryBatch({ + id: delivery.id, + batchMemberIds: delivery.batchMemberIds?.map( + (id) => new mongoose.Types.ObjectId(String(id)), + ), + }), + }; +} + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + methods = createMethods(mongoose); + storage = createResponseStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +afterEach(async () => { + await service?.stop(); + service = undefined; +}); +beforeEach(async () => { + await runAsSystem(async () => { + await Promise.all( + [ + 'StoredResponse', + 'ResponseScope', + 'AgentTriggerDelivery', + 'AgentTriggerLaneSequence', + 'AgentTriggerUserPurge', + ].map((name) => mongoose.models[name].deleteMany({})), + ); + }); +}); + +async function reserve(principal: ResponseOwner = owner): Promise { + const result = await storage.reserve({ + ...principal, + responseId: 'resp_queue', + requestKey: 'key', + fingerprint: 'request', + agentId: 'agent', + sourceKeyId: 'original-api-key', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'Original input', + instructions: 'Preserve instructions', + background: true, + store: true, + }), + conversationId: 'conversation', + parentMessageId: 'root', + }); + if (!result.ok) throw new Error('Reservation failed'); + return result.record; +} + +it('executes queued Responses through its registered runtime, never through chat HTTP', () => + scope(async () => { + const executed: StoredResponseRecord[] = []; + const fetcher = jest.fn(async () => { + throw new Error('Chat transport must not run'); + }); + const queue = createBackgroundResponseQueue({ + storage, + enqueue: (...args) => service!.enqueue(...args), + authorize: async (record) => record.sourceKeyId === 'original-api-key', + execute: async (record) => { + executed.push(record); + return { status: 'started', generationCreatedAt: 100 }; + }, + }); + service = createAgentTriggerService({ + methods: persistence(), + sourceDispatchers: queue.sourceDispatchers, + fetch: fetcher, + deliveryOptions: { tickMs: 10 }, + }); + await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } }); + const reserved = await reserve(); + const attached = await storage.attachDelivery( + owner, + reserved.responseId, + queue.deliveryKey(reserved), + ); + if (!attached.ok) throw new Error('Attach failed'); + await queue.enqueue(attached.record); + for (let attempt = 0; attempt < 100 && executed.length === 0; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(executed).toHaveLength(1); + expect(JSON.parse(executed[0].requestJSON)).toMatchObject({ + instructions: 'Preserve instructions', + input: 'Original input', + }); + expect(fetcher).not.toHaveBeenCalled(); + const delivery = await service.getDelivery(attached.record.deliveryKey!); + expect(delivery?.requiredWorkerCapability).toBe('responses_background_v2'); + expect(JSON.stringify(delivery?.envelope)).not.toContain('original-api-key'); + expect(JSON.stringify(delivery?.envelope)).not.toContain('Original input'); + })); + +it('does not let a queue worker without the Responses capability claim the response', () => + scope(async () => { + const fetcher = jest.fn(async () => { + throw new Error('Old worker must not run'); + }); + service = createAgentTriggerService({ + methods: persistence(), + fetch: fetcher, + deliveryOptions: { tickMs: 10 }, + }); + await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } }); + const queue = createBackgroundResponseQueue({ + storage, + enqueue: (...args) => service!.enqueue(...args), + authorize: async () => true, + execute: async () => { + throw new Error('Not installed'); + }, + }); + const record = await reserve(); + const attached = await storage.attachDelivery( + owner, + record.responseId, + queue.deliveryKey(record), + ); + if (!attached.ok) throw new Error('Attach failed'); + await queue.enqueue(attached.record); + for (const workerCapabilities of [[], ['responses_background_v1']]) { + const claimed = await runAsSystem(async () => + methods.claimNextAgentTriggerDelivery({ + workerId: 'old-worker', + claimToken: 'old-token', + now: new Date(), + leaseUntil: new Date(Date.now() + 1000), + workerCapabilities, + }), + ); + expect(claimed).toBeNull(); + } + expect(fetcher).not.toHaveBeenCalled(); + })); + +it('rejects a tampered source envelope and revoked original authority before execution', () => + scope(async () => { + const execute = jest.fn(async () => ({ status: 'started' as const })); + let authorized = true; + const queue = createBackgroundResponseQueue({ + storage, + enqueue: async () => { + throw new Error('Not used'); + }, + authorize: async () => authorized, + execute, + }); + const record = await reserve(); + await storage.attachDelivery(owner, record.responseId, queue.deliveryKey(record)); + const dispatch = queue.sourceDispatchers.get('responses')!.dispatch; + const expected = createBackgroundResponseEnvelope(record); + await expect(dispatch({ ...expected, input: 'tampered' })).rejects.toMatchObject({ + code: 'response_authority_changed', + }); + authorized = false; + await expect(dispatch(expected)).rejects.toMatchObject({ code: 'response_authority_changed' }); + expect(execute).not.toHaveBeenCalled(); + })); + +async function reserveDecision(): Promise { + await reserve(); + await storage.publish(owner, 'resp_queue', 0, { + status: 'in_progress', + generationCreatedAt: 100, + responseJSON: '{}', + }); + const paused = await storage.publishReview( + owner, + 'resp_queue', + 1, + 100, + { + actionId: 'action-one', + providerExecutionId: 'provider-one', + expiresAt: Date.now() + 60_000, + partialResponseJSON: '{"private":"unfinished output"}', + }, + '{}', + ); + if (!paused.ok) throw new Error('Pause failed'); + const result = await storage.reserveResume( + owner, + 'resp_queue', + { + generationCreatedAt: 100, + actionId: 'action-one', + intentId: 'intent-one', + previousProviderExecutionId: 'provider-one', + providerExecutionId: 'provider-two', + fingerprint: 'decision-one', + resolutionJSON: '{"answer":"private owner answer"}', + }, + Date.now(), + ); + if (!result.ok) throw new Error('Decision reservation failed'); + return result.record; +} + +it('deduplicates saved decisions in the existing queue and excludes old Responses workers', () => + scope(async () => { + const fetcher = jest.fn(async () => { + throw new Error('Chat must not run'); + }); + service = createAgentTriggerService({ + methods: persistence(), + fetch: fetcher, + deliveryOptions: { tickMs: 10 }, + }); + await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } }); + const queue = createResponseResumeQueue({ + storage, + enqueue: (...args) => service!.enqueue(...args), + authorize: async () => true, + resume: async () => { + throw new Error('Resume host is not installed'); + }, + }); + const record = await reserveDecision(); + await Promise.all(Array.from({ length: 8 }, () => queue.enqueue(record))); + const delivery = await service.getDelivery(queue.deliveryKey(record)); + expect(delivery?.requiredWorkerCapability).toBe('responses_resume_v1'); + expect(await mongoose.models.AgentTriggerDelivery.countDocuments({})).toBe(1); + const serialized = JSON.stringify(delivery?.envelope); + for (const secret of [ + 'private owner answer', + 'unfinished output', + 'original-api-key', + 'Original input', + ]) + expect(serialized).not.toContain(secret); + for (const workerCapabilities of [ + [], + ['responses_background_v1'], + ['responses_background_v2'], + ]) { + expect( + await runAsSystem(() => + methods.claimNextAgentTriggerDelivery({ + workerId: 'old-worker', + claimToken: 'old-token', + now: new Date(), + leaseUntil: new Date(Date.now() + 1000), + workerCapabilities, + }), + ), + ).toBeNull(); + } + const claimed = await runAsSystem(() => + methods.claimNextAgentTriggerDelivery({ + workerId: 'resume-worker', + claimToken: 'resume-token', + now: new Date(), + leaseUntil: new Date(Date.now() + 1000), + workerCapabilities: ['responses_resume_v1'], + }), + ); + expect(claimed).not.toBeNull(); + expect(fetcher).not.toHaveBeenCalled(); + })); + +it('rechecks exact saved decision and source grant before dispatching the resume checkpoint', () => + scope(async () => { + const execute = jest.fn(async () => ({ status: 'resumed' as const, generationCreatedAt: 100 })); + let authorized = true; + const queue = createResponseResumeQueue({ + storage, + enqueue: async () => { + throw new Error('Not used'); + }, + authorize: async () => authorized, + resume: execute, + }); + const record = await reserveDecision(); + const envelope = createResponseResumeEnvelope(record); + const dispatch = queue.sourceDispatchers.get(BACKGROUND_RESPONSE_RESUME_SOURCE)!.dispatch; + await expect(dispatch({ ...envelope, input: 'tampered' })).rejects.toMatchObject({ + code: 'response_resume_authority_changed', + }); + await expect( + dispatch({ ...envelope, event: { ...envelope.event, id: 'other-intent' } }), + ).rejects.toMatchObject({ code: 'response_resume_authority_changed' }); + authorized = false; + await expect(dispatch(envelope)).rejects.toMatchObject({ + code: 'response_resume_authority_changed', + }); + expect(execute).not.toHaveBeenCalled(); + authorized = true; + expect(await dispatch(envelope)).toMatchObject({ + status: 'resumed', + streamId: record.responseId, + generationCreatedAt: 100, + }); + expect(execute).toHaveBeenCalledTimes(1); + expect(execute).toHaveBeenCalledWith( + expect.objectContaining({ resume: expect.objectContaining({ intentId: 'intent-one' }) }), + undefined, + ); + })); + +it('never starts a cancelled decision and does not confuse queue settlement with provider drain', () => + scope(async () => { + const execute = jest.fn(async () => ({ status: 'resumed' as const })); + const queue = createResponseResumeQueue({ + storage, + enqueue: async () => { + throw new Error('Not used'); + }, + authorize: async () => true, + resume: execute, + }); + const record = await reserveDecision(); + await storage.requestCancellation(owner, record.responseId, Date.now()); + const dispatch = queue.sourceDispatchers.get(BACKGROUND_RESPONSE_RESUME_SOURCE)!.dispatch; + expect(await dispatch(createResponseResumeEnvelope(record))).toMatchObject({ + status: 'settled', + }); + expect(execute).not.toHaveBeenCalled(); + expect((await storage.read(owner, record.responseId))?.status).toBe('requires_action'); + })); + +it('rejects an old delivery after a newer provider segment has paused for a different action', () => + scope(async () => { + const execute = jest.fn(async () => ({ status: 'resumed' as const })); + const queue = createResponseResumeQueue({ + storage, + enqueue: async () => { + throw new Error('Not used'); + }, + authorize: async () => true, + resume: execute, + }); + const record = await reserveDecision(); + const admitted = await storage.admitResume( + owner, + record.responseId, + 'intent-one', + 100, + 'provider-two', + '{}', + Date.now(), + ); + if (!admitted.ok) throw new Error('Admission failed'); + await storage.publishReview( + owner, + record.responseId, + admitted.record.revision, + 100, + { + actionId: 'action-two', + providerExecutionId: 'provider-two', + expiresAt: Date.now() + 60_000, + partialResponseJSON: '{}', + }, + '{}', + ); + const dispatch = queue.sourceDispatchers.get(BACKGROUND_RESPONSE_RESUME_SOURCE)!.dispatch; + await expect(dispatch(createResponseResumeEnvelope(record))).rejects.toMatchObject({ + code: 'response_resume_authority_changed', + }); + expect(execute).not.toHaveBeenCalled(); + })); + +it.each([null, ''])( + 'dispatches a legacy tenantless stored response %p through the actual trigger queue', + async (tenantId) => { + const principal = { userId: owner.userId }; + await tenantStorage.run(principal, async () => { + const execute = jest.fn().mockResolvedValue({ status: 'started', generationCreatedAt: 100 }); + const queue = createBackgroundResponseQueue({ + storage, + enqueue: (...args) => service!.enqueue(...args), + authorize: async () => true, + execute, + }); + service = createAgentTriggerService({ + methods: persistence(), + sourceDispatchers: queue.sourceDispatchers, + deliveryOptions: { tickMs: 10 }, + }); + await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } }); + const saved = await reserve(principal); + const key = queue.deliveryKey(saved); + await mongoose.models.StoredResponse.collection.updateOne( + { responseId: saved.responseId }, + { $set: { tenantId } }, + ); + const legacy = await storage.read(principal, saved.responseId); + if (!legacy) throw new Error('Legacy fixture missing'); + expect(queue.deliveryKey(legacy)).toBe(key); + const attached = await storage.attachDelivery(principal, legacy.responseId, key); + if (!attached.ok) throw new Error('Attach failed'); + await queue.enqueue(attached.record); + for (let attempt = 0; attempt < 100 && execute.mock.calls.length === 0; attempt++) + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(execute).toHaveBeenCalledTimes(1); + expect(execute.mock.calls[0][0]).toMatchObject({ + userId: principal.userId, + tenantId, + responseId: saved.responseId, + }); + }); + }, +); diff --git a/packages/api/src/agents/responses/queue.ts b/packages/api/src/agents/responses/queue.ts new file mode 100644 index 0000000000..84f24f700a --- /dev/null +++ b/packages/api/src/agents/responses/queue.ts @@ -0,0 +1,240 @@ +import { + AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_V2, + AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1, +} from '@librechat/data-schemas'; +import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; +import type { AgentTriggerExecutionHost } from '../triggers/host'; +import type { AgentTriggerEnvelope } from '../triggers/envelope'; +import type { AgentTriggerService } from '../triggers/service'; +import { + createAgentTriggerEnvelope, + getAgentTriggerIdempotencyKey, + parseAgentTriggerEnvelope, +} from '../triggers/envelope'; +import { prepareAgentTriggerDelivery } from '../triggers/delivery'; +import { responseTenantId, isResponseTenantId } from './scope'; +import { AgentTriggerExecutionError } from '../triggers/host'; + +export const BACKGROUND_RESPONSE_SOURCE = 'responses'; +export const BACKGROUND_RESPONSE_RESUME_SOURCE = 'responses.resume'; + +/** A saved decision selects this identity once; HTTP/queue retries reuse it verbatim. */ +export function createResponseResumeEnvelope(record: StoredResponseRecord): AgentTriggerEnvelope { + const intent = record.resume; + if (!intent || record.generationCreatedAt == null || !isResponseTenantId(record.tenantId)) + throw new Error('Response resume intent is unavailable'); + const identity = `${record.responseId}:resume:${intent.intentId}`; + return createAgentTriggerEnvelope({ + mode: 'fire', + requestId: identity, + deliveryId: identity, + receivedAt: intent.reservedAt, + principal: { id: record.userId, tenantId: responseTenantId(record.tenantId) }, + target: { agentId: record.agentId }, + run: { conversationId: record.conversationId }, + event: { + id: intent.intentId, + type: 'response.resume', + occurredAt: intent.reservedAt, + source: { id: record.responseId, type: BACKGROUND_RESPONSE_RESUME_SOURCE }, + }, + input: 'Resume the saved Responses checkpoint.', + }); +} + +/** Install only when this host implements same-checkpoint resume, never as a chat fallback. */ +export function createResponseResumeQueue( + deps: Omit & { + /** Admit only the saved intent's provider segment within its original generation epoch. */ + resume: BackgroundResponseQueueDependencies['execute']; + }, +): BackgroundResponseQueue { + const deliveryKey = (record: StoredResponseRecord) => + getAgentTriggerIdempotencyKey(createResponseResumeEnvelope(record)); + const dispatch: AgentTriggerExecutionHost['dispatch'] = async (input, options) => { + const envelope = parseAgentTriggerEnvelope(input); + const invalid = () => + new AgentTriggerExecutionError('Response resume authority changed', { + mode: envelope.mode, + certainty: 'definite', + retryable: false, + code: 'response_resume_authority_changed', + status: 403, + }); + if ( + envelope.event.source.type !== BACKGROUND_RESPONSE_RESUME_SOURCE || + envelope.mode !== 'fire' + ) + throw invalid(); + const record = await deps.storage.read(envelope.principal, envelope.event.source.id); + if (!record?.resume || !(await deps.authorize(record))) throw invalid(); + if ( + prepareAgentTriggerDelivery(envelope).fingerprint !== + prepareAgentTriggerDelivery(createResponseResumeEnvelope(record)).fingerprint + ) + throw invalid(); + const identity = { + mode: 'fire' as const, + conversationId: record.conversationId, + streamId: record.responseId, + responseMessageId: record.responseId, + generationCreatedAt: record.generationCreatedAt, + }; + // A persisted decision receipt survives cancellation; it must never restart the provider. + if ( + record.cancellationRequestedAt != null || + ['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status) + ) + return { ...identity, status: 'settled' }; + if (!['requires_action', 'in_progress'].includes(record.status)) throw invalid(); + const execution = await deps.resume(record, options?.signal); + return { ...identity, ...execution }; + }; + return { + deliveryKey, + sourceDispatchers: new Map([ + [ + BACKGROUND_RESPONSE_RESUME_SOURCE, + { + workerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1, + dispatch, + }, + ], + ]), + async enqueue(record) { + const envelope = createResponseResumeEnvelope(record); + const receipt = await deps.enqueue(envelope, { + orderingKey: `responses:${responseTenantId(record.tenantId) ?? '_'}:${record.userId}:${record.conversationId}`, + requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1, + }); + if (receipt.deliveryKey !== getAgentTriggerIdempotencyKey(envelope)) + throw new Error('Response resume queue receipt identity changed'); + }, + }; +} + +/** The queue carries an immutable response reference, never credentials or a second request body. */ +export function createBackgroundResponseEnvelope( + record: StoredResponseRecord, +): AgentTriggerEnvelope { + if (!isResponseTenantId(record.tenantId)) + throw new Error('Response tenant authority unavailable'); + return createAgentTriggerEnvelope({ + mode: 'fire', + requestId: record.responseId, + deliveryId: record.responseId, + receivedAt: record.createdAt.getTime(), + principal: { id: record.userId, tenantId: responseTenantId(record.tenantId) }, + target: { agentId: record.agentId }, + run: { conversationId: record.conversationId }, + event: { + id: record.responseId, + type: 'response.create', + occurredAt: record.createdAt.getTime(), + source: { id: record.responseId, type: BACKGROUND_RESPONSE_SOURCE }, + }, + input: 'Execute the saved Responses request.', + }); +} + +export function backgroundResponseDeliveryKey(record: StoredResponseRecord): string { + return getAgentTriggerIdempotencyKey(createBackgroundResponseEnvelope(record)); +} + +export interface BackgroundResponseQueueDependencies { + storage: ResponseStorage; + enqueue: AgentTriggerService['enqueue']; + authorize: (record: StoredResponseRecord) => Promise; + workerCapability?: string; + /** Execute the persisted Responses payload with its exact stable identity. + * Admission MUST be idempotent: a repeated delivery attaches rather than starting a second run. + * The runtime also owns cancellation-before-start and already terminal reconciliation. */ + execute: ( + record: StoredResponseRecord, + signal?: AbortSignal, + ) => Promise<{ + status: 'started' | 'resumed' | 'settled'; + generationCreatedAt?: number; + }>; +} + +export interface BackgroundResponseQueue { + deliveryKey: (record: StoredResponseRecord) => string; + sourceDispatchers: ReadonlyMap< + string, + { + workerCapability: string; + dispatch: AgentTriggerExecutionHost['dispatch']; + } + >; + enqueue: (record: StoredResponseRecord) => Promise; +} + +export function createBackgroundResponseQueue( + deps: BackgroundResponseQueueDependencies, +): BackgroundResponseQueue { + const dispatch: AgentTriggerExecutionHost['dispatch'] = async (input, options) => { + const envelope = parseAgentTriggerEnvelope(input); + const invalid = () => + new AgentTriggerExecutionError('Background response authority changed', { + mode: envelope.mode, + certainty: 'definite', + retryable: false, + code: 'response_authority_changed', + status: 403, + }); + if (envelope.event.source.type !== BACKGROUND_RESPONSE_SOURCE || envelope.mode !== 'fire') + throw invalid(); + const record = await deps.storage.read(envelope.principal, envelope.event.source.id); + if (!record || !(await deps.authorize(record))) throw invalid(); + const expected = createBackgroundResponseEnvelope(record); + if ( + prepareAgentTriggerDelivery(envelope).fingerprint !== + prepareAgentTriggerDelivery(expected).fingerprint || + record.deliveryKey !== backgroundResponseDeliveryKey(record) + ) + throw invalid(); + if (['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status)) { + return { + mode: 'fire', + status: 'settled', + conversationId: record.conversationId, + streamId: record.responseId, + responseMessageId: record.responseId, + generationCreatedAt: record.generationCreatedAt, + }; + } + const execution = await deps.execute(record, options?.signal); + return { + mode: 'fire', + ...execution, + conversationId: record.conversationId, + streamId: record.responseId, + responseMessageId: record.responseId, + }; + }; + + return { + deliveryKey: backgroundResponseDeliveryKey, + sourceDispatchers: new Map([ + [ + BACKGROUND_RESPONSE_SOURCE, + { + workerCapability: deps.workerCapability ?? AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_V2, + dispatch, + }, + ], + ]), + async enqueue(record: StoredResponseRecord) { + if (record.deliveryKey !== backgroundResponseDeliveryKey(record)) + throw new Error('Response queue identity changed'); + const receipt = await deps.enqueue(createBackgroundResponseEnvelope(record), { + orderingKey: `responses:${responseTenantId(record.tenantId) ?? '_'}:${record.userId}:${record.conversationId}`, + requiredWorkerCapability: + deps.workerCapability ?? AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_V2, + }); + if (receipt.deliveryKey !== record.deliveryKey) + throw new Error('Response queue receipt identity changed'); + }, + }; +} diff --git a/packages/api/src/agents/responses/recovery.spec.ts b/packages/api/src/agents/responses/recovery.spec.ts new file mode 100644 index 0000000000..9c157dbcce --- /dev/null +++ b/packages/api/src/agents/responses/recovery.spec.ts @@ -0,0 +1,630 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createResponseStorage, + createModels, + createMethods, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { + ResponseStorage, + StoredResponseRecord, + ResponseTerminalReceipt, +} from '@librechat/data-schemas'; +import { createBackgroundResponseService, buildPendingBackgroundResponse } from './background'; +import { createCheckpointNamespace } from '~/stream/checkpoints'; +import { cleanupResponseCheckpoint } from './checkpoint'; +import { createResponseCompletion } from './completion'; +import { createResponseRetirement } from './retirement'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +const owner = { userId: new mongoose.Types.ObjectId().toString(), tenantId: 'completion-tenant' }; +const scoped = (work: () => Promise) => tenantStorage.run(owner, work); +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createResponseStorage(mongoose); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + await mongoose.models.User.deleteMany({}); + }); + await scoped(async () => { + await mongoose.models.User.create({ + _id: owner.userId, + tenantId: owner.tenantId, + email: 'completion-owner@example.com', + provider: 'local', + }); + }); +}); +afterEach(() => { + jest.restoreAllMocks(); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +async function setup(): Promise<{ + record: StoredResponseRecord; + receipt: ResponseTerminalReceipt; +}> { + const reserved = await storage.reserve({ + ...owner, + responseId: 'resp_completion', + requestKey: 'original-request', + fingerprint: 'original', + agentId: 'agent', + sourceKeyId: 'original-key', + conversationId: 'conversation', + parentMessageId: 'root', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'Original input', + background: true, + store: true, + }), + }); + if (!reserved.ok) throw new Error('Reservation failed'); + const initial = { + ...buildPendingBackgroundResponse(reserved.record), + status: 'in_progress' as const, + }; + const running = await storage.publish(owner, reserved.record.responseId, 0, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: JSON.stringify(initial), + }); + if (!running.ok) throw new Error('Publication failed'); + const response = { + ...initial, + status: 'completed' as const, + output: [ + { + type: 'message', + id: 'message', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'saved final answer', annotations: [] }], + }, + ], + }; + return { + record: running.record, + receipt: { + generationCreatedAt: 10, + providerExecutionId: 'provider', + ownerId: 'winning-worker', + status: 'completed', + responseJSON: JSON.stringify(response), + checkpointNamespace: 'owned-scope', + }, + }; +} + +it('retains the exact completed answer across cleanup failure and a fresh recovery instance', () => + scoped(async () => { + const { record, receipt } = await setup(); + const cleanup = jest.fn(async () => {}); + cleanup.mockRejectedValueOnce(new Error('checkpoint unavailable')); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: cleanup, + }); + await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow( + 'checkpoint unavailable', + ); + const pending = (await storage.read(owner, record.responseId))!; + expect(pending).toMatchObject({ status: 'in_progress', pendingTerminal: receipt }); + expect(pending.responseJSON).not.toContain('saved final answer'); + const restarted = createResponseCompletion({ + releaseExecution: async () => {}, + storage: createResponseStorage(mongoose), + cleanupCheckpoint: cleanup, + }); + const complete = await restarted.finish(pending); + expect(complete).toMatchObject({ + status: 'completed', + responseJSON: receipt.responseJSON, + requestKey: 'original-request', + }); + expect(complete.pendingTerminal).toBeUndefined(); + expect(cleanup).toHaveBeenCalledTimes(2); + })); + +it('recovers initial-execution checkpoint cleanup without a review record or current deployment settings', () => + scoped(async () => { + const { record, receipt } = await setup(); + const namespace = createCheckpointNamespace(owner.userId, owner.tenantId); + const sibling = createCheckpointNamespace(owner.userId, owner.tenantId); + const settings = { + type: 'mongo', + checkpointCollectionName: 'initial_receipt_checkpoints', + checkpointWritesCollectionName: 'initial_receipt_writes', + }; + const savedReceipt = { + ...receipt, + checkpointNamespace: namespace, + checkpointConfigJSON: JSON.stringify(settings), + }; + const collections = [ + settings.checkpointCollectionName, + settings.checkpointWritesCollectionName, + ]; + for (const name of collections) { + const collection = mongoose.connection.db!.collection(name); + await collection.deleteMany({}); + await collection.insertMany( + [namespace, sibling].map((scope) => ({ + thread_id: record.conversationId, + checkpoint_ns: scope, + checkpoint_id: scope, + })), + ); + } + const unavailable = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => { + throw new Error('checkpoint storage temporarily unavailable'); + }, + }); + await expect(unavailable.stageAndFinish(record, savedReceipt)).rejects.toThrow( + 'temporarily unavailable', + ); + const pending = (await storage.read(owner, record.responseId))!; + expect(pending.review).toBeUndefined(); + expect(pending.pendingTerminal).toEqual(savedReceipt); + const restarted = createResponseCompletion({ + releaseExecution: async () => {}, + storage: createResponseStorage(mongoose), + cleanupCheckpoint: cleanupResponseCheckpoint, + }); + expect((await restarted.finish(pending)).status).toBe('completed'); + for (const name of collections) { + const remaining = await mongoose.connection.db!.collection(name).find({}).toArray(); + expect(remaining).toHaveLength(1); + expect(remaining[0].checkpoint_ns).toBe(sibling); + } + })); + +it('retries terminal publication without losing the answer after cleanup already succeeded', () => + scoped(async () => { + const { record, receipt } = await setup(); + const cleanup = jest.fn(async () => {}); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: cleanup, + }); + jest + .spyOn(storage, 'finishTerminal') + .mockRejectedValueOnce(new Error('publication unavailable')); + await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow( + 'publication unavailable', + ); + const pending = (await storage.read(owner, record.responseId))!; + expect(pending.pendingTerminal?.responseJSON).toBe(receipt.responseJSON); + expect((await completion.finish(pending)).responseJSON).toBe(receipt.responseJSON); + expect(cleanup).toHaveBeenCalledTimes(2); + })); + +it('reconciles a lost finish acknowledgement and retries execution release on a later terminal read', () => + scoped(async () => { + const { record, receipt } = await setup(); + const finish = storage.finishTerminal.bind(storage); + jest.spyOn(storage, 'finishTerminal').mockImplementationOnce(async (...args) => { + await finish(...args); + throw new Error('finish acknowledgement lost'); + }); + const release = jest.fn(async () => {}); + release.mockRejectedValueOnce(new Error('release transport unavailable')); + const completion = createResponseCompletion({ + storage, + cleanupCheckpoint: async () => {}, + releaseExecution: release, + }); + await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow( + 'release transport unavailable', + ); + const published = (await storage.read(owner, record.responseId))!; + expect(published.status).toBe('completed'); + expect(published.pendingTerminal).toBeUndefined(); + expect((await completion.finish(published)).responseJSON).toBe(receipt.responseJSON); + expect(release).toHaveBeenCalledTimes(2); + })); + +it('reconciles a committed terminal stage whose acknowledgement was lost', () => + scoped(async () => { + const { record, receipt } = await setup(); + const stage = storage.stageTerminal.bind(storage); + const spy = jest.spyOn(storage, 'stageTerminal').mockImplementationOnce(async (...args) => { + await stage(...args); + throw new Error('lost stage acknowledgement'); + }); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + expect(await completion.stageAndFinish(record, receipt)).toMatchObject({ + status: 'completed', + responseJSON: receipt.responseJSON, + }); + expect(spy).toHaveBeenCalledTimes(1); + })); + +it.each([false, true])( + 'stages by execution identity despite projection changes after provider drain (late Stop: %s)', + (lateStop) => + scoped(async () => { + const { record, receipt } = await setup(); + const projected = await storage.publish(owner, record.responseId, record.revision, { + status: 'in_progress', + responseJSON: JSON.stringify({ + ...JSON.parse(record.responseJSON!), + recovery: 'execution_state_unavailable', + }), + }); + expect(projected.ok).toBe(true); + if (lateStop) await storage.requestCancellation(owner, record.responseId, Date.now()); + const stage = jest.spyOn(storage, 'stageTerminal'); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + const result = await completion.stageAndFinish(record, receipt); + expect(result).toMatchObject({ status: 'completed', responseJSON: receipt.responseJSON }); + expect(stage).toHaveBeenCalledTimes(1); + }), +); + +it('does not replace another terminal receipt after a staging conflict', () => + scoped(async () => { + const { record, receipt } = await setup(); + const competing = { ...receipt, ownerId: 'another-worker' }; + expect( + (await storage.stageTerminal(owner, record.responseId, record.revision, competing)).ok, + ).toBe(true); + const stage = jest.spyOn(storage, 'stageTerminal'); + const cleanup = jest.fn(async () => {}); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: cleanup, + }); + await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow( + 'Response terminal receipt publication conflicted', + ); + expect(stage).toHaveBeenCalledTimes(1); + expect(cleanup).not.toHaveBeenCalled(); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(competing); + })); + +it('does not retry a receipt from a different generation', () => + scoped(async () => { + const { record, receipt } = await setup(); + const stage = jest.spyOn(storage, 'stageTerminal'); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + await expect( + completion.stageAndFinish( + { ...record, generationCreatedAt: 9 }, + { ...receipt, generationCreatedAt: 9 }, + ), + ).rejects.toThrow('Response terminal receipt publication conflicted'); + expect(stage).toHaveBeenCalledTimes(1); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined(); + })); + +it.each(['provider', 'another-provider'])( + 'does not retry across a new review checkpoint owned by %s', + (providerExecutionId) => + scoped(async () => { + const { record, receipt } = await setup(); + const reviewed = await storage.publishReview( + owner, + record.responseId, + record.revision, + receipt.generationCreatedAt, + { + actionId: 'new-action', + providerExecutionId, + expiresAt: Date.now() + 60_000, + partialResponseJSON: record.responseJSON!, + }, + record.responseJSON!, + ); + expect(reviewed.ok).toBe(true); + const stage = jest.spyOn(storage, 'stageTerminal'); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + await expect(completion.stageAndFinish(record, receipt)).rejects.toThrow( + 'Response terminal receipt publication conflicted', + ); + expect(stage).toHaveBeenCalledTimes(1); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined(); + }), +); + +it.each([false, true])( + 'does not retry a terminal receipt after a new resume intent (admitted: %s)', + (admitted) => + scoped(async () => { + const { record, receipt } = await setup(); + const reviewed = await storage.publishReview( + owner, + record.responseId, + record.revision, + receipt.generationCreatedAt, + { + actionId: 'action', + providerExecutionId: receipt.providerExecutionId, + expiresAt: Date.now() + 60_000, + partialResponseJSON: record.responseJSON!, + }, + record.responseJSON!, + ); + if (!reviewed.ok) throw new Error('Review publication failed'); + const reserved = await storage.reserveResume( + owner, + record.responseId, + { + generationCreatedAt: receipt.generationCreatedAt, + intentId: 'intent', + actionId: 'action', + previousProviderExecutionId: receipt.providerExecutionId, + providerExecutionId: 'next-provider', + fingerprint: 'decision', + resolutionJSON: '{}', + }, + Date.now(), + ); + expect(reserved.ok).toBe(true); + if (admitted) + expect( + ( + await storage.admitResume( + owner, + record.responseId, + 'intent', + receipt.generationCreatedAt, + 'next-provider', + record.responseJSON!, + Date.now(), + ) + ).ok, + ).toBe(true); + const stage = jest.spyOn(storage, 'stageTerminal'); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + await expect(completion.stageAndFinish(reviewed.record, receipt)).rejects.toThrow( + 'Response terminal receipt publication conflicted', + ); + expect(stage).toHaveBeenCalledTimes(1); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined(); + }), +); + +it('lets authorized polling recover a saved result without queueing or observing execution', () => + scoped(async () => { + const { record, receipt } = await setup(); + await storage.stageTerminal(owner, record.responseId, record.revision, receipt); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + const enqueue = jest.fn(async () => { + throw new Error('No model redelivery'); + }); + const observe = jest.fn(async () => { + throw new Error('Do not reconstruct progress'); + }); + const service = createBackgroundResponseService({ + storage, + authorize: async () => true, + enqueue, + observe, + resolveContinuation: async () => ({ + conversationId: 'conversation', + parentMessageId: 'root', + }), + cancel: async () => {}, + recoverTerminal: completion.finish, + }); + const result = await service.retrieve(owner, record.responseId); + expect(result).toMatchObject({ ok: true, response: JSON.parse(receipt.responseJSON) }); + expect(enqueue).not.toHaveBeenCalled(); + expect(observe).not.toHaveBeenCalled(); + })); + +it('does not let a late Stop overwrite the saved result or require a transient job to survive', () => + scoped(async () => { + const { record, receipt } = await setup(); + await storage.stageTerminal(owner, record.responseId, record.revision, receipt); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + const getJob = jest.fn(async () => { + throw new Error('Transient job may have expired'); + }); + const abortJob = jest.fn(async () => { + throw new Error('No provider needs stopping'); + }); + const retire = createResponseRetirement({ + storage, + jobs: { getJob, abortJob, finishProviderTerminalPersistence: jest.fn() }, + recoverTerminal: completion.finish, + }); + expect(await retire(owner, record.responseId)).toBe('stopped'); + expect(await storage.read(owner, record.responseId)).toMatchObject({ + status: 'completed', + responseJSON: receipt.responseJSON, + }); + expect(getJob).not.toHaveBeenCalled(); + expect(abortJob).not.toHaveBeenCalled(); + })); + +it('does not recover or disclose a saved reply after source authorization is revoked', () => + scoped(async () => { + const { record, receipt } = await setup(); + await storage.stageTerminal(owner, record.responseId, record.revision, receipt); + const recoverTerminal = jest.fn(async () => { + throw new Error('Unauthorized recovery'); + }); + const cancel = jest.fn(async () => {}); + const service = createBackgroundResponseService({ + storage, + authorize: async () => false, + enqueue: async () => {}, + observe: async () => null, + resolveContinuation: async () => ({ + conversationId: 'conversation', + parentMessageId: 'root', + }), + cancel, + recoverTerminal, + }); + const denied = { ok: false, status: 404, code: 'response_not_found' }; + expect(await service.retrieve(owner, record.responseId)).toEqual(denied); + expect(await service.cancel(owner, record.responseId)).toEqual(denied); + expect(recoverTerminal).not.toHaveBeenCalled(); + expect(cancel).not.toHaveBeenCalled(); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toEqual(receipt); + })); + +it('converges concurrent finishers on one immutable canonical result', () => + scoped(async () => { + const { record, receipt } = await setup(); + const staged = await storage.stageTerminal(owner, record.responseId, record.revision, receipt); + if (!staged.ok) throw new Error('Stage failed'); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + const results = await Promise.all( + Array.from({ length: 12 }, () => completion.finish(staged.record)), + ); + expect( + results.every( + (result) => result.status === 'completed' && result.responseJSON === receipt.responseJSON, + ), + ).toBe(true); + expect((await storage.read(owner, record.responseId))?.revision).toBe( + staged.record.revision + 1, + ); + })); + +it('rejects invalid receipt identity before it can poison the durable journal', () => + scoped(async () => { + const { record, receipt } = await setup(); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + await expect( + completion.stageAndFinish(record, { ...receipt, responseJSON: 'not JSON' }), + ).rejects.toThrow('Response terminal receipt is invalid'); + await expect( + completion.stageAndFinish(record, { ...receipt, generationCreatedAt: 11 }), + ).rejects.toThrow('Response terminal receipt identity changed'); + expect((await storage.read(owner, record.responseId))?.pendingTerminal).toBeUndefined(); + })); + +it.each(['owner', 'conversation'] as const)( + 'discards a drained terminal payload after durable %s retirement without expiry', + (kind) => + scoped(async () => { + const { record, receipt } = await setup(); + const db = createMethods(mongoose); + if (kind === 'owner') await db.deleteUserById(owner.userId); + else await db.deleteConvos(owner.userId, { conversationId: record.conversationId }); + expect(await storage.read(owner, record.responseId)).toBeNull(); + const cleanup = jest.fn(async () => {}); + const release = jest.fn(async () => {}); + const completion = createResponseCompletion({ + storage, + cleanupCheckpoint: cleanup, + releaseExecution: release, + }); + const retired = await completion.stageAndFinish(record, receipt); + expect(retired).toMatchObject({ + status: 'completed', + requestJSON: '', + retirement: kind === 'owner' ? 'owner_deleted' : 'conversation_deleted', + }); + expect(retired.responseJSON).toBeUndefined(); + expect(retired.pendingTerminal).toBeUndefined(); + expect(cleanup).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledWith( + expect.objectContaining({ + retirement: kind === 'owner' ? 'owner_deleted' : 'conversation_deleted', + }), + receipt, + ); + expect(await storage.read(owner, record.responseId)).toBeNull(); + }), +); + +it.each(['owner', 'conversation'] as const)( + 'returns not found when an authorized read races %s retirement during recovery', + (kind) => + scoped(async () => { + const { record, receipt } = await setup(); + await storage.stageTerminal(owner, record.responseId, record.revision, receipt); + const completion = createResponseCompletion({ + releaseExecution: async () => {}, + storage, + cleanupCheckpoint: async () => {}, + }); + const observe = jest.fn(); + const enqueue = jest.fn(); + const service = createBackgroundResponseService({ + storage, + authorize: async () => true, + observe, + enqueue, + resolveContinuation: async () => ({ + conversationId: record.conversationId, + parentMessageId: 'root', + }), + cancel: async () => {}, + recoverTerminal: async (retained) => { + const db = createMethods(mongoose); + if (kind === 'owner') await db.deleteUserById(owner.userId); + else await db.deleteConvos(owner.userId, { conversationId: record.conversationId }); + return completion.finish(retained); + }, + }); + const result = await service.retrieve(owner, record.responseId); + expect(result).toEqual({ ok: false, status: 404, code: 'response_not_found' }); + expect(JSON.stringify(result)).not.toContain('saved final answer'); + expect(observe).not.toHaveBeenCalled(); + expect(enqueue).not.toHaveBeenCalled(); + }), +); diff --git a/packages/api/src/agents/responses/release.spec.ts b/packages/api/src/agents/responses/release.spec.ts new file mode 100644 index 0000000000..ed8e8c9df3 --- /dev/null +++ b/packages/api/src/agents/responses/release.spec.ts @@ -0,0 +1,143 @@ +import { tenantStorage } from '@librechat/data-schemas'; +import type { StoredResponseRecord, ResponseTerminalReceipt } from '@librechat/data-schemas'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { createResponseExecutionRelease } from './completion'; + +const owner = { userId: 'release-owner', tenantId: 'release-tenant' }; +const scope = (work: () => Promise) => tenantStorage.run(owner, work); +let jobs: GenerationJobManagerClass; +beforeEach(() => { + jobs = new GenerationJobManagerClass(); + jobs.configure({ + jobStore: new InMemoryJobStore(), + eventTransport: new InMemoryEventTransport(), + isRedis: false, + }); + jobs.initialize(); +}); +afterEach(async () => { + await jobs.destroy(); + jest.restoreAllMocks(); +}); + +async function fixture() { + const responseId = 'resp_release'; + const job = await jobs.createJob(responseId, owner.userId, 'conversation', { + initialMetadata: { endpoint: 'responses', agent_id: 'agent', responseMessageId: responseId }, + }); + const providerExecutionId = job.metadata.providerExecutionId!; + await jobs.beginProviderExecution(responseId, job.createdAt, providerExecutionId); + await jobs.completeJob(responseId, undefined, job.createdAt, { + expectedProviderExecutionId: providerExecutionId, + persistencePending: true, + }); + await jobs.markProviderExecutionDrained(responseId, job.createdAt, providerExecutionId, true); + const record: StoredResponseRecord = { + ...owner, + responseId, + agentId: 'agent', + sourceKeyId: 'source', + requestKey: 'request', + requestJSON: '{}', + fingerprint: 'fingerprint', + conversationId: 'conversation', + parentMessageId: 'parent', + status: 'completed', + generationCreatedAt: job.createdAt, + revision: 1, + createdAt: new Date(), + updatedAt: new Date(), + responseJSON: '{}', + }; + const receipt: ResponseTerminalReceipt = { + generationCreatedAt: job.createdAt, + providerExecutionId, + status: 'completed', + responseJSON: '{}', + }; + expect((await jobs.getJob(responseId))?.metadata.terminalPersistencePending).toBe(true); + return { record, receipt, release: createResponseExecutionRelease(jobs) }; +} + +it('releases exact post-publication terminal proof with production default cleanup enabled', () => + scope(async () => { + const f = await fixture(); + await f.release(f.record, f.receipt); + expect(await jobs.getJob(f.record.responseId)).toBeUndefined(); + await expect(f.release(f.record, f.receipt)).resolves.toBeUndefined(); + })); + +it('recovers cleanup when Mongo already published and removed its pending receipt', () => + scope(async () => { + const f = await fixture(); + expect(f.record.pendingTerminal).toBeUndefined(); + await f.release(f.record); + expect(await jobs.getJob(f.record.responseId)).toBeUndefined(); + })); + +it.each([ + { userId: 'another-owner' }, + { tenantId: 'another-tenant' }, + { agentId: 'another-agent' }, + { conversationId: 'another-conversation' }, + { generationCreatedAt: 1 }, +])('does not release another execution for mismatched record %j', (change) => + scope(async () => { + const f = await fixture(); + await expect(f.release({ ...f.record, ...change }, f.receipt)).rejects.toThrow( + 'identity changed', + ); + expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).toBe( + true, + ); + }), +); + +it('does not release proof for another provider attempt', () => + scope(async () => { + const f = await fixture(); + await expect( + f.release(f.record, { ...f.receipt, providerExecutionId: 'another-provider' }), + ).rejects.toThrow('identity changed'); + expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).toBe( + true, + ); + })); + +it('does not treat a rejected release acknowledgement as success', () => + scope(async () => { + const f = await fixture(); + jest.spyOn(jobs, 'finishProviderTerminalPersistence').mockResolvedValueOnce(false); + await expect(f.release(f.record, f.receipt)).rejects.toThrow('release is unavailable'); + expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).toBe( + true, + ); + await f.release(f.record, f.receipt); + expect(await jobs.getJob(f.record.responseId)).toBeUndefined(); + })); + +it('retries job cleanup after the marker was cleared but deletion failed', () => + scope(async () => { + const f = await fixture(); + jest + .spyOn(jobs.getJobStore(), 'deleteJob') + .mockRejectedValueOnce(new Error('transient deletion failure')); + await expect(f.release(f.record, f.receipt)).rejects.toThrow('transient deletion failure'); + expect((await jobs.getJob(f.record.responseId))?.metadata.terminalPersistencePending).not.toBe( + true, + ); + await f.release(f.record); + expect(await jobs.getJob(f.record.responseId)).toBeUndefined(); + })); + +it.each(['owner_deleted', 'conversation_deleted', 'expired'] as const)( + 'releases only exact drained proof for a durable %s cleanup identity', + (retirement) => + scope(async () => { + const f = await fixture(); + await f.release({ ...f.record, status: 'in_progress', retirement }, f.receipt); + expect(await jobs.getJob(f.record.responseId)).toBeUndefined(); + }), +); diff --git a/packages/api/src/agents/responses/retirement.spec.ts b/packages/api/src/agents/responses/retirement.spec.ts new file mode 100644 index 0000000000..d2e32632df --- /dev/null +++ b/packages/api/src/agents/responses/retirement.spec.ts @@ -0,0 +1,374 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createResponseStorage, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { ResponseStorage, ResponseOwner } from '@librechat/data-schemas'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { createResponseCompletion, createResponseExecutionRelease } from './completion'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { buildPendingBackgroundResponse } from './background'; +import { enrollAgentExecution } from '../remote/lifecycle'; +import { createResponseRetirement } from './retirement'; + +const owner = { userId: new mongoose.Types.ObjectId().toString(), tenantId: 'stop-tenant' }; +const scoped = (work: () => Promise) => tenantStorage.run(owner, work); +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +let jobs: GenerationJobManagerClass; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createResponseStorage(mongoose); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); + await scoped(async () => { + await mongoose.models.User.deleteMany({}); + await mongoose.models.User.create({ + _id: owner.userId, + tenantId: owner.tenantId, + email: 'stop-owner@example.com', + provider: 'local', + }); + }); + jobs = new GenerationJobManagerClass(); + jobs.configure({ + jobStore: new InMemoryJobStore(), + eventTransport: new InMemoryEventTransport(), + }); + jobs.initialize(); +}); +afterEach(async () => { + jest.restoreAllMocks(); + await jobs.destroy(); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +async function fixture(principal: ResponseOwner = owner, retainTerminalPersistence = true) { + const responseId = 'resp_running_stop'; + const execution = await enrollAgentExecution( + { + runId: responseId, + userId: principal.userId, + conversationId: 'conversation', + agentId: 'agent', + protocol: 'responses', + ...(retainTerminalPersistence ? { retainTerminalPersistence: true as const } : {}), + isPrincipalActive: async () => true, + }, + { manager: jobs }, + ); + const reserved = await storage.reserve({ + ...principal, + responseId, + requestKey: 'original-request', + fingerprint: 'original', + agentId: 'agent', + sourceKeyId: 'original-key', + conversationId: 'conversation', + parentMessageId: 'root', + requestJSON: JSON.stringify({ model: 'agent', input: 'Test', background: true, store: true }), + }); + if (!reserved.ok) throw new Error('Reservation failed'); + const running = await storage.publish(principal, responseId, 0, { + status: 'in_progress', + generationCreatedAt: execution.createdAt, + responseJSON: JSON.stringify({ + ...buildPendingBackgroundResponse(reserved.record), + status: 'in_progress', + }), + }); + if (!running.ok) throw new Error('Publication failed'); + await execution.beginProviderExecution(); + const completion = createResponseCompletion({ + storage, + cleanupCheckpoint: async () => {}, + releaseExecution: createResponseExecutionRelease(jobs), + }); + const retire = createResponseRetirement({ + storage, + jobs, + recoverTerminal: completion.finish, + }); + /** Force the live failure ordering: provider settles before abort's chat publication. */ + const abort = jobs.abortJob.bind(jobs); + jest.spyOn(jobs, 'abortJob').mockImplementationOnce((id, options) => + abort(id, { + ...options, + beforePublish: async (result) => { + expect(execution.signal.aborted).toBe(true); + await execution.settle(); + expect((await jobs.getJob(id))?.metadata.providerDrained).toBe(true); + await options?.beforePublish?.(result); + }, + }), + ); + return { execution, responseId, retire, completion }; +} + +it('retains a running Stop after provider drain until canonical cancellation releases it with default cleanup', () => + scoped(async () => { + const f = await fixture(); + const publishChat = jest.spyOn(jobs, 'publishTerminalClaim'); + const stage = storage.stageTerminal.bind(storage); + jest.spyOn(storage, 'stageTerminal').mockImplementationOnce(async (...args) => { + expect(await jobs.getJob(f.responseId)).toMatchObject({ + status: 'aborted', + createdAt: f.execution.createdAt, + metadata: { providerDrained: true, terminalPersistencePending: true }, + }); + expect(await storage.read(owner, f.responseId)).toMatchObject({ + status: 'in_progress', + cancellationRequestedAt: expect.any(Number), + }); + return stage(...args); + }); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + expect(publishChat).not.toHaveBeenCalled(); + expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled'); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + })); + +it('drains expired work through metadata identity without publishing private cancellation output', () => + scoped(async () => { + const f = await fixture(); + await mongoose.models.StoredResponse.updateOne( + { responseId: f.responseId }, + { + $set: { expiredAt: new Date(Date.now() - 1) }, + }, + ); + expect(await storage.read(owner, f.responseId)).toBeNull(); + const stage = jest.spyOn(storage, 'stageTerminal'); + const release = jest.spyOn(jobs, 'finishProviderTerminalPersistence'); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + expect(stage).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledTimes(1); + expect(f.execution.signal.aborted).toBe(true); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + expect(await storage.read(owner, f.responseId)).toBeNull(); + })); + +it('keeps drained Stop proof through canonical receipt failure and retry without executing again', () => + scoped(async () => { + const f = await fixture(); + jest.spyOn(storage, 'stageTerminal').mockRejectedValue(new Error('receipt unavailable')); + await expect(f.retire(owner, f.responseId)).rejects.toThrow('receipt unavailable'); + expect(await jobs.getJob(f.responseId)).toMatchObject({ + status: 'aborted', + metadata: { providerDrained: true, terminalPersistencePending: true }, + }); + jest.mocked(storage.stageTerminal).mockRestore(); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled'); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + })); + +it('drains through the retained journal after conversation erasure closes admission', () => + scoped(async () => { + const f = await fixture(); + createModels(mongoose); + await mongoose.models.Conversation.create({ + conversationId: 'conversation', + user: owner.userId, + tenantId: owner.tenantId, + endpoint: 'agents', + }); + const db = createMethods(mongoose); + await db.deleteConvos( + owner.userId, + { conversationId: 'conversation' }, + { + beforeDelete: async () => { + expect((await storage.read(owner, f.responseId))?.cancellationRequestedAt).toEqual( + expect.any(Number), + ); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + }, + }, + ); + expect(await storage.read(owner, f.responseId)).toBeNull(); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + })); + +it('does not relabel an already-completed provider as cancelled before its canonical publication', () => + scoped(async () => { + const f = await fixture(); + await f.execution.settle(); + const stage = jest.spyOn(storage, 'stageTerminal'); + expect(await f.retire(owner, f.responseId)).toBe('pending'); + expect(stage).not.toHaveBeenCalled(); + expect((await jobs.getJob(f.responseId))?.status).toBe('complete'); + expect((await storage.read(owner, f.responseId))?.status).toBe('in_progress'); + })); + +it('repeated Stop recovers a running runtime receipt failure without a paused review or a new execution', () => + scoped(async () => { + const f = await fixture(); + await storage.requestCancellation(owner, f.responseId, Date.now()); + await jobs.abortJob(f.responseId, { + expectedCreatedAt: f.execution.createdAt, + awaitProviderDrain: true, + retainTerminalPersistence: true, + }); + const identity = await f.execution.confirmTerminalIdentity(); + expect(identity.outcome).toBe('aborted'); + const record = (await storage.read(owner, f.responseId))!; + expect(record.review).toBeUndefined(); + jest.spyOn(storage, 'stageTerminal').mockRejectedValueOnce(new Error('runtime receipt failed')); + await expect( + f.completion.stageAndFinish(record, { + ...identity, + status: 'cancelled', + responseJSON: JSON.stringify({ + ...buildPendingBackgroundResponse(record), + status: 'cancelled', + }), + }), + ).rejects.toThrow('runtime receipt failed'); + expect((await storage.read(owner, f.responseId))?.pendingTerminal).toBeUndefined(); + expect((await jobs.getJob(f.responseId))?.metadata.terminalPersistencePending).toBe(true); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled'); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + })); + +it('preserves ordinary abort publication and default cleanup when retention is not requested', () => + scoped(async () => { + const f = await fixture(owner, false); + const publishChat = jest.spyOn(jobs, 'publishTerminalClaim'); + expect(await jobs.abortJob(f.responseId, { awaitProviderDrain: true })).toMatchObject({ + success: true, + finalEvent: { aborted: true }, + }); + expect(publishChat).toHaveBeenCalledTimes(1); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + await expect(f.execution.confirmTerminalIdentity()).rejects.toThrow( + 'terminal settlement could not be confirmed', + ); + })); + +it('rejects retained abort on unsupported stores before changing the owned generation', () => + scoped(async () => { + const f = await fixture(); + Object.defineProperty(jobs.getJobStore(), 'providerTerminalPersistence', { + value: false, + configurable: true, + }); + await expect(f.retire(owner, f.responseId)).rejects.toThrow('barrier is unavailable'); + expect((await jobs.getJob(f.responseId))?.status).toBe('running'); + expect(f.execution.signal.aborted).toBe(false); + Object.defineProperty(jobs.getJobStore(), 'providerTerminalPersistence', { value: true }); + await f.execution.settle(); + })); + +it('releases retained provider proof through the complete fallback recovery', () => + scoped(async () => { + const f = await fixture(); + const retire = createResponseRetirement({ storage, jobs, cleanupCheckpoint: async () => {} }); + expect(await retire(owner, f.responseId)).toBe('stopped'); + expect((await storage.read(owner, f.responseId))?.status).toBe('cancelled'); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + expect(await retire(owner, f.responseId)).toBe('stopped'); + })); + +it('keeps the fallback retention barrier until failed checkpoint cleanup is retried', () => + scoped(async () => { + const f = await fixture(); + await f.execution.settle(); + const record = (await storage.read(owner, f.responseId))!; + const job = (await jobs.getJob(f.responseId))!; + const receipt = { + generationCreatedAt: f.execution.createdAt, + providerExecutionId: job.metadata.providerExecutionId!, + ownerId: job.metadata.providerExecutionOwnerId, + status: 'completed' as const, + responseJSON: JSON.stringify({ + ...buildPendingBackgroundResponse(record), + status: 'completed', + }), + checkpointNamespace: 'owned-scope', + checkpointConfigJSON: '{"pinned":true}', + }; + expect( + (await storage.stageTerminal(owner, f.responseId, { execution: record }, receipt)).ok, + ).toBe(true); + const cleanup = jest.fn(async () => {}); + cleanup.mockRejectedValueOnce(new Error('checkpoint unavailable')); + const retire = createResponseRetirement({ storage, jobs, cleanupCheckpoint: cleanup }); + await expect(retire(owner, f.responseId)).rejects.toThrow('checkpoint unavailable'); + expect((await jobs.getJob(f.responseId))?.metadata.terminalPersistencePending).toBe(true); + expect((await storage.read(owner, f.responseId))?.pendingTerminal).toEqual(receipt); + expect(await retire(owner, f.responseId)).toBe('stopped'); + expect(cleanup).toHaveBeenCalledTimes(2); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + })); + +it.each(['owner', 'conversation'] as const)( + 'drains a retained provider after %s erasure without an expiry deadline', + (kind) => + scoped(async () => { + const f = await fixture(); + const db = createMethods(mongoose); + if (kind === 'owner') await db.deleteUserById(owner.userId); + else await db.deleteConvos(owner.userId, { conversationId: 'conversation' }); + expect(await storage.read(owner, f.responseId)).toBeNull(); + const job = (await jobs.getJob(f.responseId))!; + expect( + await storage.readCleanupIdentity(owner, f.responseId, { + ...owner, + responseId: f.responseId, + agentId: 'agent', + conversationId: 'conversation', + generationCreatedAt: job.createdAt, + }), + ).toMatchObject({ retirement: kind === 'owner' ? 'owner_deleted' : 'conversation_deleted' }); + expect(await f.retire(owner, f.responseId)).toBe('stopped'); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + expect(await storage.read(owner, f.responseId)).toBeNull(); + }), +); + +it.each([null, ''])( + 'retires and releases real provider state for legacy persisted tenant %p', + async (tenantId) => { + const principal = { userId: new mongoose.Types.ObjectId().toString() }; + await tenantStorage.run(principal, async () => { + await mongoose.models.User.create({ + _id: principal.userId, + email: 'legacy-stop@example.com', + provider: 'local', + }); + const f = await fixture(principal); + await mongoose.models.StoredResponse.collection.updateOne( + { responseId: f.responseId, userId: principal.userId }, + { $set: { tenantId } }, + ); + const record = await storage.read(principal, f.responseId); + expect(record?.tenantId).toBe(tenantId); + expect((await jobs.getJob(f.responseId))?.metadata.tenantId).toBeUndefined(); + expect(await f.retire(principal, f.responseId)).toBe('stopped'); + expect((await storage.read(principal, f.responseId))?.status).toBe('cancelled'); + expect(await jobs.getJob(f.responseId)).toBeUndefined(); + expect(await f.retire(principal, f.responseId)).toBe('stopped'); + }); + }, +); diff --git a/packages/api/src/agents/responses/retirement.ts b/packages/api/src/agents/responses/retirement.ts new file mode 100644 index 0000000000..bed9311cdb --- /dev/null +++ b/packages/api/src/agents/responses/retirement.ts @@ -0,0 +1,256 @@ +import type { ResponseOwner, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; +import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import type { ResponseCheckpointIdentity } from './checkpoint'; +import type { ResponseCompletionRecord } from './completion'; +import { createResponseCompletion, createResponseExecutionRelease } from './completion'; +import { buildPendingBackgroundResponse } from './background'; +import { responseTenantId } from './scope'; + +export interface ResponseRetirementDependencies { + storage: ResponseStorage; + jobs: Pick< + GenerationJobManagerClass, + 'getJob' | 'abortJob' | 'finishProviderTerminalPersistence' + >; + /** Required before a durably paused response may publish cancellation. Must be idempotent. */ + cleanupCheckpoint?: ( + record: ResponseCheckpointIdentity, + checkpointNamespace: string, + ) => Promise; + recoverTerminal?: (record: StoredResponseRecord) => Promise; + now?: () => number; +} + +const terminal = (record: StoredResponseRecord) => + ['completed', 'failed', 'incomplete', 'cancelled'].includes(record.status); + +/** Internal cleanup authority: may stop one owned response, never execute or disclose its output. */ +export function createResponseRetirement( + deps: ResponseRetirementDependencies, +): (owner: ResponseOwner, responseId: string) => Promise<'stopped' | 'pending'> { + const recover = + deps.recoverTerminal ?? + createResponseCompletion({ + storage: deps.storage, + cleanupCheckpoint: + deps.cleanupCheckpoint ?? + (async () => { + throw new Error('Response checkpoint cleanup unavailable'); + }), + releaseExecution: createResponseExecutionRelease(deps.jobs), + }).finish; + return async (owner, responseId) => { + let record = await deps.storage.requestCancellation( + owner, + responseId, + (deps.now ?? Date.now)(), + ); + if (!record) { + const job = await deps.jobs.getJob(responseId); + const retained = + job && + job.metadata.userId === owner.userId && + responseTenantId(job.metadata.tenantId) === responseTenantId(owner.tenantId) && + job.metadata.endpoint === 'responses' && + job.metadata.responseMessageId === responseId && + job.metadata.agent_id && + job.metadata.conversationId + ? { + userId: job.metadata.userId, + tenantId: job.metadata.tenantId, + responseId, + agentId: job.metadata.agent_id, + conversationId: job.metadata.conversationId, + generationCreatedAt: job.createdAt, + checkpointNamespace: job.metadata.checkpointNamespace, + checkpointConfigJSON: job.metadata.responseCheckpointConfigJSON, + } + : undefined; + const identity = await deps.storage.readCleanupIdentity(owner, responseId, retained); + if ( + !identity?.retirement && + !(identity?.expiredAt && identity.expiredAt.getTime() <= (deps.now ?? Date.now)()) + ) + throw new Error('Response retirement identity unavailable'); + if (!job) return identity.generationCreatedAt == null ? 'stopped' : 'pending'; + if ( + job.createdAt !== identity.generationCreatedAt || + job.metadata.userId !== identity.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(identity.tenantId) || + job.metadata.agent_id !== identity.agentId || + job.metadata.endpoint !== 'responses' || + job.metadata.conversationId !== identity.conversationId || + job.metadata.responseMessageId !== responseId + ) + throw new Error('Response retirement generation changed'); + const providerExecutionId = job.metadata.providerExecutionId; + const providerExecutionOwnerId = job.metadata.providerExecutionOwnerId; + if (!providerExecutionId) throw new Error('Response retirement provider unavailable'); + const cleanup = async () => { + if (identity.requiresCheckpointCleanup || identity.checkpointConfigJSON) { + const namespace = identity.checkpointNamespace ?? job.metadata.checkpointNamespace; + if (!namespace || !deps.cleanupCheckpoint || !identity.checkpointConfigJSON) + throw new Error('Response checkpoint cleanup unavailable'); + await deps.cleanupCheckpoint( + { + ...identity, + pendingTerminal: { + checkpointNamespace: namespace, + checkpointConfigJSON: identity.checkpointConfigJSON, + }, + }, + namespace, + ); + } + }; + await deps.jobs.abortJob(responseId, { + expectedCreatedAt: identity.generationCreatedAt, + awaitProviderDrain: true, + retainTerminalPersistence: true, + beforePublish: cleanup, + }); + await createResponseExecutionRelease({ + getJob: deps.jobs.getJob.bind(deps.jobs), + finishProviderTerminalPersistence: deps.jobs.finishProviderTerminalPersistence.bind( + deps.jobs, + ), + })( + { ...identity, status: 'cancelled' }, + { providerExecutionId, ownerId: providerExecutionOwnerId }, + ); + return 'stopped'; + } + if (terminal(record)) { + await recover(record); + return 'stopped'; + } + if (record.pendingTerminal) { + return terminal(await recover(record)) ? 'stopped' : 'pending'; + } + if (record.generationCreatedAt == null) { + const response = { + ...buildPendingBackgroundResponse(record), + status: 'cancelled' as const, + completed_at: Math.floor((deps.now ?? Date.now)() / 1000), + }; + const cancelled = await deps.storage.publish(owner, responseId, record.revision, { + status: 'cancelled', + responseJSON: JSON.stringify(response), + }); + // This CAS beats enrollment publication, which must succeed before provider startup. + if (cancelled.ok) return 'stopped'; + record = await deps.storage.read(owner, responseId); + if (!record) throw new Error('Response disappeared during retirement'); + if (terminal(record)) { + await recover(record); + return 'stopped'; + } + } + const job = await deps.jobs.getJob(responseId); + if (!job) return 'pending'; + if ( + record.generationCreatedAt == null || + job.createdAt !== record.generationCreatedAt || + job.metadata.userId !== record.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(record.tenantId) || + job.metadata.agent_id !== record.agentId || + job.metadata.endpoint !== 'responses' || + job.metadata.conversationId !== record.conversationId || + job.metadata.responseMessageId !== record.responseId + ) + throw new Error('Response retirement generation changed'); + const epoch = record.generationCreatedAt; + const agentId = record.agentId; + const conversationId = record.conversationId; + const providerExecutionId = job.metadata.providerExecutionId; + const providerExecutionOwnerId = job.metadata.providerExecutionOwnerId; + async function stageCancellation(): Promise { + const stopped = await deps.jobs.getJob(responseId); + const proof = stopped?.metadata; + if ( + stopped?.status !== 'aborted' || + !proof || + proof.providerDrained !== true || + stopped.createdAt !== epoch || + proof.userId !== owner.userId || + responseTenantId(proof.tenantId) !== responseTenantId(owner.tenantId) || + proof.agent_id !== agentId || + proof.endpoint !== 'responses' || + proof.responseMessageId !== responseId || + proof.conversationId !== conversationId || + proof.providerExecutionId !== providerExecutionId || + proof.providerExecutionOwnerId !== providerExecutionOwnerId || + !proof.providerExecutionId + ) + return; + const current = await deps.storage.read(owner, responseId); + if (!current || current.generationCreatedAt !== epoch) + throw new Error('Response retirement journal changed'); + if (terminal(current) || current.pendingTerminal) return; + if (current.cancellationRequestedAt == null) + throw new Error('Response cancellation intent is unavailable'); + if (current.review && (!proof.checkpointNamespace || !deps.cleanupCheckpoint)) return; + const response = { + ...buildPendingBackgroundResponse(current), + model: current.agentId, + status: 'cancelled' as const, + completed_at: Math.floor((deps.now ?? Date.now)() / 1000), + }; + const receipt = { + generationCreatedAt: epoch, + providerExecutionId: proof.providerExecutionId, + ...(proof.providerExecutionOwnerId == null + ? {} + : { ownerId: proof.providerExecutionOwnerId }), + status: 'cancelled' as const, + responseJSON: JSON.stringify(response), + ...(current.review + ? { + checkpointNamespace: proof.checkpointNamespace, + ...(current.review.checkpointConfigJSON == null + ? {} + : { checkpointConfigJSON: current.review.checkpointConfigJSON }), + } + : {}), + }; + try { + const staged = await deps.storage.stageTerminal( + owner, + responseId, + { execution: current }, + receipt, + ); + if (!staged.ok) throw new Error('Response retirement receipt conflicted'); + } catch (error) { + const saved = await deps.storage.read(owner, responseId); + if ( + !saved || + saved.generationCreatedAt !== epoch || + (!terminal(saved) && + (saved.pendingTerminal?.providerExecutionId !== receipt.providerExecutionId || + saved.pendingTerminal.responseJSON !== receipt.responseJSON)) + ) + throw error; + } + } + await deps.jobs.abortJob(responseId, { + expectedCreatedAt: record.generationCreatedAt, + awaitProviderDrain: true, + retainTerminalPersistence: true, + beforePublish: stageCancellation, + }); + // Runtime publication includes provider drain and trailing writes; a missing job alone is not proof. + let settled = await deps.storage.read(owner, responseId); + if (settled && terminal(settled)) { + await recover(settled); + return 'stopped'; + } + if (settled?.pendingTerminal) return terminal(await recover(settled)) ? 'stopped' : 'pending'; + // A repeated Stop can recover a failed runtime receipt only from the exact retained, + // aborted and drained provider. Completed jobs and absent jobs are not cancellation proof. + await stageCancellation(); + settled = await deps.storage.read(owner, responseId); + if (settled?.pendingTerminal) settled = await recover(settled); + return settled && terminal(settled) ? 'stopped' : 'pending'; + }; +} diff --git a/packages/api/src/agents/responses/scope.ts b/packages/api/src/agents/responses/scope.ts new file mode 100644 index 0000000000..af9980dac6 --- /dev/null +++ b/packages/api/src/agents/responses/scope.ts @@ -0,0 +1,14 @@ +import { isReservedTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas'; + +/** Legacy persisted tenantless rows use missing, null or empty values. */ +export function responseTenantId(value: string | null | undefined): string | undefined { + return value || undefined; +} + +/** Internal partition sentinels never acquire response execution authority. */ +export function isResponseTenantId(value: unknown): value is string | null | undefined { + return ( + value == null || + (typeof value === 'string' && value !== SYSTEM_TENANT_ID && !isReservedTenantId(value)) + ); +} diff --git a/packages/api/src/agents/responses/transport.spec.ts b/packages/api/src/agents/responses/transport.spec.ts new file mode 100644 index 0000000000..06288033ff --- /dev/null +++ b/packages/api/src/agents/responses/transport.spec.ts @@ -0,0 +1,172 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createResponseStorage, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; +import type { BackgroundResponseTransportDependencies } from './transport'; +import { createBackgroundResponseTransport } from './transport'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +let record: StoredResponseRecord; +const owner = { userId: '000000000000000000000001', tenantId: 'transport-test' }; +type Fetch = NonNullable; +const scoped = (work: () => Promise) => tenantStorage.run(owner, work); +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createResponseStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); + const saved = await scoped(() => + storage.reserve({ + ...owner, + responseId: 'resp_test', + agentId: 'agent', + sourceKeyId: 'original-key', + requestKey: 'request', + fingerprint: 'fingerprint', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'Private input', + background: true, + store: true, + }), + conversationId: 'conversation', + parentMessageId: 'root', + }), + ); + if (!saved.ok) throw new Error('Reservation failed'); + record = saved.record; +}); +async function enroll() { + const result = await scoped(() => + storage.publish(owner, record.responseId, record.revision, { + status: 'in_progress', + responseJSON: JSON.stringify(ack), + generationCreatedAt: 1234, + }), + ); + if (!result.ok) throw new Error('Enrollment failed'); +} +function transport(fetch: Fetch, origin = 'http://127.0.0.1:3088') { + return createBackgroundResponseTransport({ + storage, + fetch, + getOrigin: () => origin, + signingSecret: () => 'transport-test-secret', + acknowledgementTimeoutMs: async () => 1000, + }); +} +const ack = { + id: 'resp_test', + object: 'response', + model: 'agent', + status: 'in_progress', + background: true, + store: true, +}; + +it('sends only a server-purpose credential and saved reference through Responses ingress', async () => { + const fetch: Fetch = async (url, init) => { + expect(String(url)).toBe('http://127.0.0.1:3088/api/agents/v1/responses'); + expect(init?.body).toBe('{}'); + expect(init?.headers).toMatchObject({ 'x-lc-response-worker': '1' }); + expect(JSON.stringify(init)).not.toContain('Private input'); + await enroll(); + return Response.json(ack, { status: 202 }); + }; + await expect(scoped(() => transport(fetch)(record))).resolves.toEqual({ + status: 'started', + generationCreatedAt: 1234, + }); +}); + +it('recovers a lost acknowledgement only when durable enrollment proves execution', async () => { + const fetch: Fetch = async () => { + await enroll(); + throw new Error('lost ACK'); + }; + await expect(scoped(() => transport(fetch)(record))).resolves.toEqual({ + status: 'resumed', + generationCreatedAt: 1234, + }); +}); + +it('does not treat a network timeout without enrollment as successful dispatch', async () => { + const fetch: Fetch = async () => { + throw new Error('timeout'); + }; + await expect(scoped(() => transport(fetch)(record))).rejects.toMatchObject({ + code: 'response_acknowledgement_unavailable', + certainty: 'ambiguous', + }); +}); + +it('attaches duplicate admission to the same enrolled generation', async () => { + const fetch: Fetch = async () => { + await enroll(); + return new Response('', { status: 409 }); + }; + await expect(scoped(() => transport(fetch)(record))).resolves.toEqual({ + status: 'resumed', + generationCreatedAt: 1234, + }); +}); + +it.each([401, 403, 429, 503])( + 'classifies admission HTTP %s without exposing its body', + async (status) => { + const fetch: Fetch = async () => new Response('PRIVATE upstream failure', { status }); + await expect(scoped(() => transport(fetch)(record))).rejects.toMatchObject({ + message: 'Background response admission rejected', + status, + retryable: status === 429 || status >= 500, + certainty: status >= 500 ? 'ambiguous' : 'definite', + }); + }, +); + +it.each(['not JSON', JSON.stringify({ ...ack, id: 'other-response' }), JSON.stringify(ack)])( + 'does not accept an unproved or mismatched acknowledgement', + async (body) => { + const fetch: Fetch = async () => new Response(body, { status: 202 }); + await expect(scoped(() => transport(fetch)(record))).rejects.toMatchObject({ + code: 'response_acknowledgement_invalid', + }); + }, +); + +it('refuses remote or credential-bearing worker origins before making a request', async () => { + let calls = 0; + const fetch: Fetch = async () => { + calls++; + return Response.json(ack); + }; + for (const origin of [ + 'https://example.com', + 'http://user:secret@127.0.0.1:3088', + 'file:///tmp/test', + ]) { + await expect(scoped(() => transport(fetch, origin)(record))).rejects.toThrow( + 'capable local listener', + ); + } + expect(calls).toBe(0); +}); diff --git a/packages/api/src/agents/responses/transport.ts b/packages/api/src/agents/responses/transport.ts new file mode 100644 index 0000000000..93f2a3f562 --- /dev/null +++ b/packages/api/src/agents/responses/transport.ts @@ -0,0 +1,189 @@ +import { z } from 'zod'; +import type { StoredResponseRecord, ResponseStorage } from '@librechat/data-schemas'; +import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import type { BackgroundResponseQueueDependencies } from './queue'; +import { + mintBackgroundResponseWorkerToken, + mintResponseResumeWorkerToken, + RESPONSE_WORKER_TOKEN_EXPIRED, +} from './worker'; +import { AgentTriggerExecutionError } from '../triggers/host'; +import { responseTenantId } from './scope'; + +export interface BackgroundResponseTransportDependencies { + storage: ResponseStorage; + getOrigin: () => string; + signingSecret: () => string; + acknowledgementTimeoutMs: (record: StoredResponseRecord) => Promise; + fetch?: ( + input: Parameters[0], + init?: Parameters[1], + ) => ReturnType; +} + +const expiredWorkerCredential = z.object({ + error: z.object({ code: z.literal(RESPONSE_WORKER_TOKEN_EXPIRED) }), +}); + +const acknowledgement = z.object({ + id: z.string(), + object: z.literal('response'), + model: z.string(), + background: z.literal(true), + store: z.literal(true), + status: z.enum(['queued', 'in_progress', 'completed', 'cancelled', 'failed', 'incomplete']), +}); + +export function createBackgroundResponseTransport( + deps: BackgroundResponseTransportDependencies, +): BackgroundResponseQueueDependencies['execute'] { + return createResponseWorkerTransport(deps); +} + +/** A consumed approval alone is not proof that its selected provider attempt started. */ +export function createResponseResumeTransport( + deps: BackgroundResponseTransportDependencies & { + jobs: Pick; + }, +): BackgroundResponseQueueDependencies['execute'] { + return createResponseWorkerTransport(deps, deps.jobs); +} + +/** Only the capable local replica may execute this queue delivery, through Responses ingress. */ +function createResponseWorkerTransport( + deps: BackgroundResponseTransportDependencies, + resumeJobs?: Pick, +): BackgroundResponseQueueDependencies['execute'] { + async function receipt(record: StoredResponseRecord, current: StoredResponseRecord | null) { + if (current?.generationCreatedAt == null) return; + if (!resumeJobs) + return { status: 'resumed' as const, generationCreatedAt: current.generationCreatedAt }; + if ( + !record.resume || + current.generationCreatedAt !== record.generationCreatedAt || + current.resume?.intentId !== record.resume.intentId || + current.resume.providerExecutionId !== record.resume.providerExecutionId || + current.sourceKeyId !== record.sourceKeyId || + current.agentId !== record.agentId || + current.conversationId !== record.conversationId + ) + return; + const identity = { generationCreatedAt: current.generationCreatedAt }; + if (['completed', 'failed', 'incomplete', 'cancelled'].includes(current.status)) + return { ...identity, status: 'settled' as const }; + if (current.resume.state !== 'admitted') return; + const job = await resumeJobs.getJob(record.responseId); + if ( + !job || + job.createdAt !== current.generationCreatedAt || + job.metadata.userId !== current.userId || + responseTenantId(job.metadata.tenantId) !== responseTenantId(current.tenantId) || + job.metadata.agent_id !== current.agentId || + job.metadata.conversationId !== current.conversationId || + job.metadata.responseMessageId !== current.responseId || + job.metadata.endpoint !== 'responses' || + job.metadata.providerExecutionId !== current.resume.providerExecutionId || + job.metadata.providerExecutionStartedId !== current.resume.providerExecutionId || + !job.metadata.providerExecutionOwnerId + ) + return; + return { ...identity, status: 'resumed' as const }; + } + return async ( + record: StoredResponseRecord, + signal?: AbortSignal, + ): Promise<{ + status: 'started' | 'resumed' | 'settled'; + generationCreatedAt?: number; + }> => { + const origin = new URL(deps.getOrigin()); + if ( + !['http:', 'https:'].includes(origin.protocol) || + origin.username || + origin.password || + !['127.0.0.1', '::1', '[::1]', 'localhost'].includes(origin.hostname) + ) + throw new Error('Background Responses requires the capable local listener'); + const timeoutMs = await deps.acknowledgementTimeoutMs(record); + const timeout = AbortSignal.timeout(timeoutMs); + const token = resumeJobs + ? mintResponseResumeWorkerToken(record, deps.signingSecret()) + : mintBackgroundResponseWorkerToken(record, deps.signingSecret()); + let response: globalThis.Response; + try { + response = await (deps.fetch ?? globalThis.fetch)( + new URL('/api/agents/v1/responses', origin), + { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'x-lc-response-worker': '1', + }, + body: '{}', + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + }, + ); + } catch { + const current = await deps.storage.read(record, record.responseId); + const confirmed = await receipt(record, current); + if (confirmed) return confirmed; + throw new AgentTriggerExecutionError( + 'Background response admission acknowledgement unavailable', + { + mode: 'fire', + certainty: 'ambiguous', + retryable: true, + code: 'response_acknowledgement_unavailable', + }, + ); + } + const current = await deps.storage.read(record, record.responseId); + const confirmed = await receipt(record, current); + if (response.status === 409 && confirmed) { + await response.body?.cancel(); + return confirmed; + } + if (resumeJobs && response.status === 409) { + await response.body?.cancel(); + throw new AgentTriggerExecutionError('Response resume start acknowledgement unavailable', { + mode: 'fire', + certainty: 'ambiguous', + retryable: true, + code: 'response_resume_start_unconfirmed', + }); + } + if (!response.ok) { + const expired = + response.status === 401 && + expiredWorkerCredential.safeParse(await response.json().catch(() => null)).success; + // A 401 body has already been consumed; cancelling its locked reader would hide the + // classified authentication outcome with a stream error. + if (response.status !== 401) await response.body?.cancel(); + throw new AgentTriggerExecutionError('Background response admission rejected', { + mode: 'fire', + certainty: response.status >= 500 ? 'ambiguous' : 'definite', + retryable: expired || response.status >= 500 || response.status === 429, + code: expired ? RESPONSE_WORKER_TOKEN_EXPIRED : 'response_admission_rejected', + status: response.status, + }); + } + const body = acknowledgement.safeParse(await response.json().catch(() => null)); + if ( + !body.success || + body.data.id !== record.responseId || + body.data.model !== record.agentId || + !confirmed + ) + throw new AgentTriggerExecutionError( + 'Background response acknowledgement identity unavailable', + { + mode: 'fire', + certainty: 'ambiguous', + retryable: true, + code: 'response_acknowledgement_invalid', + }, + ); + return resumeJobs ? confirmed : { ...confirmed, status: 'started' }; + }; +} diff --git a/packages/api/src/agents/responses/worker.spec.ts b/packages/api/src/agents/responses/worker.spec.ts new file mode 100644 index 0000000000..082ca3a239 --- /dev/null +++ b/packages/api/src/agents/responses/worker.spec.ts @@ -0,0 +1,352 @@ +import express from 'express'; +import jwt from 'jsonwebtoken'; +import request from 'supertest'; +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createResponseStorage, + tenantStorage, + runAsSystem, + getTenantId, +} from '@librechat/data-schemas'; +import type { ResponseStorage } from '@librechat/data-schemas'; +import type { RequestHandler } from 'express'; +import type { ServerRequest } from '~/types/http'; +import { + createBackgroundResponseWorkerAuth, + mintBackgroundResponseWorkerToken, + mintResponseResumeWorkerToken, + getBackgroundResponseWorkerRecord, + RESPONSE_WORKER_TOKEN_EXPIRED, +} from './worker'; +import { createBackgroundResponseEnvelope, createResponseResumeEnvelope } from './queue'; +import { createBackgroundResponseTransport } from './transport'; +import { buildPendingBackgroundResponse } from './background'; + +const secret = 'worker-test-secret'; +const userId = new mongoose.Types.ObjectId().toString(); +const responseId = 'resp_worker'; +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +let methods: ReturnType; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + methods = createMethods(mongoose); + storage = createResponseStorage(mongoose); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.ResponseScope.deleteMany({}); + await mongoose.models.User.deleteMany({}); + }); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +async function fixture(tenantId?: string) { + const owner = { userId, tenantId }; + return tenantStorage.run(owner, async () => { + await mongoose.models.User.create({ + _id: userId, + tenantId, + email: 'worker@example.com', + provider: 'local', + }); + const saved = await storage.reserve({ + ...owner, + responseId, + agentId: 'agent', + sourceKeyId: 'original-key', + requestKey: 'request', + fingerprint: 'fingerprint', + conversationId: 'conversation', + parentMessageId: 'root', + requestJSON: JSON.stringify({ model: 'agent', input: 'Saved input', background: true }), + }); + if (!saved.ok) throw new Error('Fixture reservation failed'); + return saved.record; + }); +} + +function app(contextTenant?: string, onWorker?: RequestHandler) { + const server = express(); + const findUser = jest.fn((id: string) => methods.findUser({ _id: id })); + const authorize = jest.fn().mockResolvedValue(true); + const onError = jest.fn(); + server.use((req, res, next) => tenantStorage.run({ tenantId: contextTenant }, next)); + server.use( + createBackgroundResponseWorkerAuth( + { storage, signingSecret: () => secret, findUser, authorize, onError }, + (req, res) => { + res.status(403).end(); + }, + ), + ); + server.post( + '/', + onWorker ?? + ((req, res) => { + const principal = (req as ServerRequest).user; + if (!principal) throw new Error('Worker principal missing'); + res.json({ + tenant: getTenantId() ?? null, + userTenant: principal.tenantId ?? null, + userId: principal.id, + body: req.body, + admitted: getBackgroundResponseWorkerRecord(req)?.responseId, + }); + }), + ); + return { server, findUser, authorize, onError }; +} + +const legacy = [undefined, null, '']; +it.each(legacy.flatMap((stored) => legacy.map((user) => [stored, user])))( + 'admits legacy stored tenant %p and user tenant %p using canonical worker claims', + async (stored, user) => { + await fixture(); + await mongoose.models.StoredResponse.collection.updateOne( + { responseId }, + stored === undefined ? { $unset: { tenantId: 1 } } : { $set: { tenantId: stored } }, + ); + await mongoose.models.User.collection.updateOne( + { _id: new mongoose.Types.ObjectId(userId) }, + user === undefined ? { $unset: { tenantId: 1 } } : { $set: { tenantId: user } }, + ); + const record = await storage.read({ userId }, responseId); + if (!record) throw new Error('Fixture disappeared'); + const token = mintBackgroundResponseWorkerToken(record, secret); + expect(jwt.decode(token)).not.toHaveProperty('tenantId'); + const f = app(); + const result = await request(f.server) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + .send({ model: 'attacker', input: 'Replaced input' }); + expect(result.status).toBe(200); + expect(result.body).toEqual({ + tenant: null, + userTenant: null, + userId, + admitted: responseId, + body: { model: 'agent', input: 'Saved input', background: true }, + }); + expect(f.authorize).toHaveBeenCalledTimes(1); + expect(f.onError).not.toHaveBeenCalled(); + const persisted = await methods.findUser({ _id: userId }); + expect(persisted?.tenantId).toBe(user); + expect(createBackgroundResponseEnvelope(record).principal.tenantId).toBeUndefined(); + const resumeRecord: Parameters[0] = { + ...record, + generationCreatedAt: 123, + resume: { + intentId: 'intent', + actionId: 'action', + previousProviderExecutionId: 'previous', + providerExecutionId: 'provider', + fingerprint: 'fingerprint', + resolutionJSON: '{}', + state: 'reserved', + reservedAt: 1, + }, + }; + const resume = mintResponseResumeWorkerToken(resumeRecord, secret); + expect(createResponseResumeEnvelope(resumeRecord).principal.tenantId).toBeUndefined(); + expect(jwt.decode(resume)).not.toHaveProperty('tenantId'); + }, +); + +it('preserves named partitions and rejects a different request tenant before database access', async () => { + const record = await fixture('tenant-a'); + const token = mintBackgroundResponseWorkerToken(record, secret); + const own = app('tenant-a'); + expect( + ( + await request(own.server) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + ).status, + ).toBe(200); + const other = app('tenant-b'); + expect( + ( + await request(other.server) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + ).status, + ).toBe(401); + expect(other.findUser).not.toHaveBeenCalled(); + expect(other.authorize).not.toHaveBeenCalled(); +}); + +it.each(['', '_', '__SYSTEM__'])('rejects noncanonical wire tenant %p', async (tenantId) => { + await fixture(); + const token = jwt.sign( + { purpose: 'responses_background', responseId, userId, tenantId, sourceKeyId: 'original-key' }, + secret, + { algorithm: 'HS256', audience: 'librechat-responses-worker', expiresIn: '60s' }, + ); + const f = app(); + expect( + ( + await request(f.server) + .post('/') + .set('x-lc-response-worker', '1') + .auth(token, { type: 'bearer' }) + ).status, + ).toBe(401); + expect(f.findUser).not.toHaveBeenCalled(); + expect(f.authorize).not.toHaveBeenCalled(); +}); + +it.each(['_', '__SYSTEM__'])( + 'refuses to mint execution authority for internal partition %p', + async (tenantId) => { + const record = await fixture(); + expect(() => mintBackgroundResponseWorkerToken({ ...record, tenantId }, secret)).toThrow(); + }, +); + +it.each(['responses_background', 'responses_resume'])( + 'reports only verified expiration as renewable for %s credentials', + async (purpose) => { + const record = await fixture(); + const token = jwt.sign( + { + purpose, + responseId: record.responseId, + userId, + sourceKeyId: record.sourceKeyId, + ...(purpose === 'responses_resume' + ? { intentId: 'intent', generationCreatedAt: 123, providerExecutionId: 'provider' } + : {}), + }, + secret, + { algorithm: 'HS256', audience: 'librechat-responses-worker', expiresIn: -1 }, + ); + const f = app(); + const result = await request(f.server) + .post('/') + .set('x-lc-response-worker', '1') + .set('Authorization', `Bearer ${token}`) + .send({}); + expect(result.status).toBe(401); + expect(result.body.error.code).toBe(RESPONSE_WORKER_TOKEN_EXPIRED); + expect(f.findUser).not.toHaveBeenCalled(); + expect(f.authorize).not.toHaveBeenCalled(); + expect(f.onError).not.toHaveBeenCalled(); + }, +); + +it.each(['signature', 'audience', 'purpose', 'claims'])( + 'keeps an expired token with invalid %s a permanent authentication denial', + async (invalid) => { + const record = await fixture(); + const token = jwt.sign( + { + purpose: invalid === 'purpose' ? 'other-worker' : 'responses_background', + responseId: invalid === 'claims' ? 42 : record.responseId, + userId, + sourceKeyId: record.sourceKeyId, + }, + invalid === 'signature' ? 'wrong-secret' : secret, + { + algorithm: 'HS256', + audience: invalid === 'audience' ? 'wrong-audience' : 'librechat-responses-worker', + expiresIn: -1, + }, + ); + const f = app(); + const result = await request(f.server) + .post('/') + .set('x-lc-response-worker', '1') + .set('Authorization', `Bearer ${token}`) + .send({}); + expect(result.status).toBe(401); + expect(result.body.error.code).toBe('invalid_response_worker'); + expect(f.findUser).not.toHaveBeenCalled(); + expect(f.authorize).not.toHaveBeenCalled(); + }, +); + +it.each([true, false])( + 'retries an expired delivery credential through real HTTP without bypassing current source policy: allowed=%s', + async (allowed) => { + const record = await fixture(); + let admissions = 0; + const f = app(undefined, async (req, res) => { + const admitted = getBackgroundResponseWorkerRecord(req); + if (!admitted) throw new Error('Worker admission missing'); + expect(req.body.input).toBe('Saved input'); + const response = { + ...buildPendingBackgroundResponse(admitted), + status: 'in_progress' as const, + }; + const saved = await storage.publish(admitted, admitted.responseId, admitted.revision, { + status: 'in_progress', + generationCreatedAt: 1234, + responseJSON: JSON.stringify(response), + }); + if (!saved.ok) throw new Error('Fixture enrollment failed'); + admissions++; + res.status(202).json(response); + }); + f.authorize.mockResolvedValue(allowed); + let clock = Date.now(); + const clockSpy = jest.spyOn(Date, 'now').mockImplementation(() => clock); + const tokens = new Set(); + let attempts = 0; + const transport = createBackgroundResponseTransport({ + storage, + getOrigin: () => 'http://127.0.0.1:3088', + signingSecret: () => secret, + acknowledgementTimeoutMs: async () => 120000, + fetch: async (_input, init) => { + attempts++; + const headers = Object.fromEntries(new Headers(init?.headers).entries()); + tokens.add(headers.authorization); + if (attempts === 1) clock += 61000; + const result = await request(f.server).post('/').set(headers).send({}); + return Response.json(result.body, { status: result.status }); + }, + }); + try { + await expect(transport(record)).rejects.toMatchObject({ + code: RESPONSE_WORKER_TOKEN_EXPIRED, + certainty: 'definite', + retryable: true, + }); + expect(admissions).toBe(0); + expect(f.authorize).not.toHaveBeenCalled(); + if (allowed) { + await expect(transport(record)).resolves.toEqual({ + status: 'started', + generationCreatedAt: 1234, + }); + expect(admissions).toBe(1); + } else { + await expect(transport(record)).rejects.toMatchObject({ + code: 'response_admission_rejected', + status: 401, + retryable: false, + }); + expect(admissions).toBe(0); + expect((await storage.read({ userId }, responseId))?.generationCreatedAt).toBeUndefined(); + } + expect(f.authorize).toHaveBeenCalledTimes(1); + expect(tokens.size).toBe(2); + } finally { + clockSpy.mockRestore(); + } + }, +); diff --git a/packages/api/src/agents/responses/worker.ts b/packages/api/src/agents/responses/worker.ts new file mode 100644 index 0000000000..c1460fdb85 --- /dev/null +++ b/packages/api/src/agents/responses/worker.ts @@ -0,0 +1,267 @@ +import { z } from 'zod'; +import jwt from 'jsonwebtoken'; +import { getTenantId, tenantStorage } from '@librechat/data-schemas'; +import type { IUser, ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; +import type { Request, RequestHandler } from 'express'; +import type { ResumedAgentExecutionIdentity } from '../remote/lifecycle'; +import type { ResponseResumeAdmissionResult } from './types'; +import type { ContextRequest } from '~/middleware/tenant'; +import type { ServerRequest } from '~/types/http'; +import { checkpointOwnerNamespacePrefix } from '~/stream/checkpoints'; +import { responseTenantId, isResponseTenantId } from './scope'; +import { tenantContextMiddleware } from '~/middleware/tenant'; + +const audience = 'librechat-responses-worker'; +const purpose = 'responses_background'; +const resumePurpose = 'responses_resume'; +export const RESPONSE_WORKER_TOKEN_EXPIRED = 'response_worker_token_expired'; +// Per-attempt security window, not a limit on durable queue work; expiration renews the credential. +const workerTokenLifetime = '60s'; +const baseClaims = z.object({ + responseId: z.string().min(1), + userId: z.string().min(1), + tenantId: z.string().min(1).refine(isResponseTenantId).optional(), + sourceKeyId: z.string().min(1), +}); +const claimsSchema = z.discriminatedUnion('purpose', [ + baseClaims.extend({ purpose: z.literal(purpose) }), + baseClaims.extend({ + purpose: z.literal(resumePurpose), + intentId: z.string().min(1).max(256), + generationCreatedAt: z.number().int().positive(), + providerExecutionId: z.string().min(1).max(128), + }), +]); +type WorkerClaims = z.infer; +type WorkerTokenError = 'invalid_response_worker' | typeof RESPONSE_WORKER_TOKEN_EXPIRED; +type WorkerTokenResult = + | { ok: true; value: WorkerClaims } + | { ok: false; error: { code: WorkerTokenError } }; + +function verifyWorkerToken(token: string, secret: string): WorkerTokenResult { + const verification: jwt.VerifyOptions = { algorithms: ['HS256'], audience }; + try { + const parsed = claimsSchema.safeParse(jwt.verify(token, secret, verification)); + if (parsed.success) return { ok: true, value: parsed.data }; + } catch (error) { + if (error instanceof jwt.TokenExpiredError) { + try { + // verify() checks expiration before audience. Revalidate every other constraint before + // treating this as a renewable credential rather than a forged or malformed request. + const expired = claimsSchema.safeParse( + jwt.verify(token, secret, { + ...verification, + ignoreExpiration: true, + }), + ); + if (expired.success) return { ok: false, error: { code: RESPONSE_WORKER_TOKEN_EXPIRED } }; + } catch { + // Invalid signatures, audiences, purposes and claim shapes remain permanent denials. + } + } + } + return { ok: false, error: { code: 'invalid_response_worker' } }; +} + +interface WorkerAdmission { + record: StoredResponseRecord; + storage: ResponseStorage; + interactive: boolean; + resume?: ResumedAgentExecutionIdentity & { intentId: string; checkpointNamespace: string }; +} +const admitted = new WeakMap(); + +export interface BackgroundResponseWorkerDependencies { + storage: ResponseStorage; + signingSecret: () => string; + findUser: (userId: string) => Promise; + /** Check original key existence/expiry, current principal, and resolved auth/Agent policy. */ + authorize: (record: StoredResponseRecord, user: IUser) => Promise; + /** Install only alongside a capable resume executor and complete current checkpoint validation. */ + admitResume?: ( + record: StoredResponseRecord, + user: IUser, + intentId: string, + ) => Promise; + onError: (error: unknown) => void; +} + +/** Distinct purpose is rejected by older initial-execution workers, not silently stripped. */ +export function mintResponseResumeWorkerToken( + record: StoredResponseRecord, + secret: string, +): string { + if ( + !record.sourceKeyId || + !secret || + !record.resume || + record.generationCreatedAt == null || + !isResponseTenantId(record.tenantId) + ) + throw new Error('Response resume worker authority unavailable'); + return jwt.sign( + { + purpose: resumePurpose, + responseId: record.responseId, + userId: record.userId, + ...(responseTenantId(record.tenantId) == null + ? {} + : { tenantId: responseTenantId(record.tenantId) }), + sourceKeyId: record.sourceKeyId, + intentId: record.resume.intentId, + generationCreatedAt: record.generationCreatedAt, + providerExecutionId: record.resume.providerExecutionId, + }, + secret, + { algorithm: 'HS256', audience, expiresIn: workerTokenLifetime }, + ); +} + +/** A worker credential is server-only, short-lived, and bound to a single API-key-authenticated response. */ +export function mintBackgroundResponseWorkerToken( + record: StoredResponseRecord, + secret: string, +): string { + if (!record.sourceKeyId || !secret || !isResponseTenantId(record.tenantId)) + throw new Error('Background response worker authority unavailable'); + return jwt.sign( + { + purpose, + responseId: record.responseId, + userId: record.userId, + ...(responseTenantId(record.tenantId) == null + ? {} + : { tenantId: responseTenantId(record.tenantId) }), + sourceKeyId: record.sourceKeyId, + }, + secret, + { algorithm: 'HS256', audience, expiresIn: workerTokenLifetime }, + ); +} + +export function getBackgroundResponseWorkerRecord(req: Request): StoredResponseRecord | undefined { + return admitted.get(req)?.record; +} + +export function getBackgroundResponseWorkerAdmission(req: Request): WorkerAdmission | undefined { + return admitted.get(req); +} + +/** Installed only on Responses ingress. Ordinary bearer authentication stays unchanged. */ +export function createBackgroundResponseWorkerAuth( + deps: BackgroundResponseWorkerDependencies, + ordinaryAuth: RequestHandler, +): RequestHandler { + return async (req, res, next) => { + if (req.headers['x-lc-response-worker'] !== '1') return ordinaryAuth(req, res, next); + const deny = (code: WorkerTokenError = 'invalid_response_worker') => { + res.status(401).json({ error: { code, message: 'Unauthorized' } }); + }; + if (req.method !== 'POST' || req.path !== '/') return deny(); + const header = req.headers.authorization; + if (!header?.startsWith('Bearer ')) return deny(); + let claims: WorkerClaims; + try { + const verified = verifyWorkerToken(header.slice(7), deps.signingSecret()); + if (!verified.ok) return deny(verified.error.code); + claims = verified.value; + } catch { + return deny(); + } + const contextTenant = getTenantId(); + if ( + responseTenantId(contextTenant) != null && + responseTenantId(contextTenant) !== claims.tenantId + ) + return deny(); + try { + await tenantStorage.run({ tenantId: claims.tenantId, userId: claims.userId }, async () => { + const [loaded, user] = await Promise.all([ + deps.storage.read(claims, claims.responseId), + deps.findUser(claims.userId), + ]); + let record = loaded; + if ( + !record || + !user || + String(user._id) !== record.userId || + !isResponseTenantId(user.tenantId) || + !isResponseTenantId(record.tenantId) || + responseTenantId(user.tenantId) !== claims.tenantId || + responseTenantId(record.tenantId) !== claims.tenantId || + record.sourceKeyId !== claims.sourceKeyId || + !(await deps.authorize(record, user)) + ) + return deny(); + // Initial delivery replay only attaches. Never let an expired generation claim + // turn an already-enrolled (possibly paused/resumed) response into a replacement job. + if (claims.purpose === purpose && record.generationCreatedAt != null) { + res.status(409).json({ code: 'RUN_ALREADY_ADMITTED' }); + return; + } + let resume: WorkerAdmission['resume']; + if (claims.purpose === resumePurpose) { + const original = record; + if ( + !deps.admitResume || + record.generationCreatedAt !== claims.generationCreatedAt || + record.resume?.intentId !== claims.intentId || + record.resume.providerExecutionId !== claims.providerExecutionId + ) + return deny(); + const result = await deps.admitResume(record, user, claims.intentId); + if (!result.ok) return deny(); + record = result.record; + if ( + record.responseId !== claims.responseId || + record.userId !== claims.userId || + !isResponseTenantId(record.tenantId) || + responseTenantId(record.tenantId) !== claims.tenantId || + record.sourceKeyId !== claims.sourceKeyId || + record.generationCreatedAt !== claims.generationCreatedAt || + record.agentId !== original.agentId || + record.conversationId !== original.conversationId || + record.requestJSON !== original.requestJSON || + record.resume?.intentId !== claims.intentId || + record.resume.providerExecutionId !== claims.providerExecutionId || + record.resume.state !== 'admitted' || + record.status !== 'in_progress' || + record.cancellationRequestedAt != null || + !result.checkpointNamespace.startsWith( + checkpointOwnerNamespacePrefix(record.userId, record.tenantId), + ) || + !(await deps.authorize(record, user)) + ) + return deny(); + resume = { + intentId: claims.intentId, + generationCreatedAt: claims.generationCreatedAt, + providerExecutionId: claims.providerExecutionId, + checkpointNamespace: result.checkpointNamespace, + }; + } + const request = req as ServerRequest & Pick; + // findUser supplies a lean auth DTO; do not mutate its cached tenant representation. + request.user = Object.assign({}, user, { id: record.userId, tenantId: claims.tenantId }); + request.tenantId = claims.tenantId; + // Caller body/headers cannot replace the persisted, already validated request. + request.body = JSON.parse(record.requestJSON); + admitted.set(req, { + record, + storage: deps.storage, + interactive: deps.admitResume != null, + ...(resume ? { resume } : {}), + }); + tenantContextMiddleware(request, res, next); + }); + } catch (error) { + deps.onError(error); + res.status(503).json({ + error: { + code: 'response_authority_unavailable', + message: 'Response authority is unavailable.', + }, + }); + } + }; +} diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index f1be3f915d..78fb62f5e8 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -3129,6 +3129,7 @@ class GenerationJobManagerClass { tenantId: jobData.tenantId, conversationId: jobData.conversationId, checkpointNamespace: jobData.checkpointNamespace, + responseCheckpointConfigJSON: jobData.responseCheckpointConfigJSON, generationProtocolVersion: jobData.generationProtocolVersion, earlyBufferOverflow: jobData.earlyBufferOverflow, userMessage: jobData.userMessage, diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts index c4bf31bdc3..1257d3b2e8 100644 --- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts +++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts @@ -338,6 +338,7 @@ describe('RedisJobStore', () => { compact: true, isTemporary: false, retentionExpiresAt: '2030-01-01T00:00:00.000Z', + responseCheckpointConfigJSON: '{"type":"mongo","checkpointCollectionName":"responses"}', agentEventDeliveryKey: 'completion-delivery-1', agentEventInvocationKey: 'original-delivery-1', agentEventInvocationGenerationCreatedAt: 987654, @@ -457,6 +458,7 @@ describe('RedisJobStore', () => { agent_id: 'agent-1', isTemporary: false, retentionExpiresAt: '2030-01-01T00:00:00.000Z', + responseCheckpointConfigJSON: '{"type":"mongo","checkpointCollectionName":"responses"}', scheduleId: 'schedule-1', scheduleMCPCompletion: { scheduleId: 'original-schedule', @@ -526,6 +528,7 @@ describe('RedisJobStore', () => { agent_id: 'agent-1', isTemporary: '0', retentionExpiresAt: '2030-01-01T00:00:00.000Z', + responseCheckpointConfigJSON: '{"type":"mongo","checkpointCollectionName":"responses"}', scheduleId: 'schedule-1', scheduledFor: '2026-08-17T12:00:00.000Z', scheduleConfigRevision: '4', diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts index 80f05ca392..7f1733c75a 100644 --- a/packages/api/src/stream/implementations/RedisJobStore.ts +++ b/packages/api/src/stream/implementations/RedisJobStore.ts @@ -5774,6 +5774,7 @@ export class RedisJobStore implements IJobStoreV2 { agent_id: data.agent_id || undefined, isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined, retentionExpiresAt: data.retentionExpiresAt || undefined, + responseCheckpointConfigJSON: data.responseCheckpointConfigJSON || undefined, agentEventDeliveryKey: data.agentEventDeliveryKey || undefined, agentEventInvocationKey: data.agentEventInvocationKey || undefined, agentEventInvocationGenerationCreatedAt: data.agentEventInvocationGenerationCreatedAt diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index f233f58ff5..93d5caaea6 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -129,6 +129,7 @@ export interface SerializableJobData { * `checkpoint_ns`, which the saver adapter maps to this storage scope. * Legacy paused jobs omit it and use the historical unscoped storage. */ checkpointNamespace?: string; + responseCheckpointConfigJSON?: string; completedAt?: number; conversationId?: string; error?: string; @@ -512,6 +513,7 @@ export type JobMetadataPatch = Partial< | 'agent_id' | 'isTemporary' | 'retentionExpiresAt' + | 'responseCheckpointConfigJSON' | 'agentEventDeliveryKey' | 'agentEventInvocationKey' | 'agentEventInvocationGenerationCreatedAt' diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts index 029bbf422c..8bfa427380 100644 --- a/packages/api/src/stream/metadata.ts +++ b/packages/api/src/stream/metadata.ts @@ -29,6 +29,8 @@ export function sanitizeJobMetadata(metadata: Partial): J patch.terminalPersistenceOwner = parseTerminalPersistenceOwner( metadata.terminalPersistenceOwner, ); + if (metadata.responseCheckpointConfigJSON !== undefined) + patch.responseCheckpointConfigJSON = metadata.responseCheckpointConfigJSON; if (metadata.responseMessageId) { patch.responseMessageId = metadata.responseMessageId; } diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts index 0390ebfdf1..7c79653b61 100644 --- a/packages/api/src/types/stream.ts +++ b/packages/api/src/types/stream.ts @@ -24,6 +24,8 @@ export interface GenerationJobMetadata { /** Immutable per-generation saver scope. LangGraph's root `checkpoint_ns` * remains empty; the checkpointer adapter maps this scope into storage. */ checkpointNamespace?: string; + /** Server-pinned saver settings retained for expired background-response cleanup. */ + responseCheckpointConfigJSON?: string; /** Immutable generation protocol. Missing on legacy records means v1. */ generationProtocolVersion?: 1 | 2; earlyBufferOverflow?: EarlyBufferOverflowState; diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 2c4cfb49a8..53af3f89bc 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -15,6 +15,7 @@ export { createResponseStorage } from './methods/response'; export type { ResponseOwner, ResponseStorage, + ResponseTerminalGuard, ResponseCleanupIdentity, ResponseReservation, ResponseReservationResult, diff --git a/packages/data-schemas/src/methods/response.spec.ts b/packages/data-schemas/src/methods/response.spec.ts index 0c62d09fb2..8a2c222a18 100644 --- a/packages/data-schemas/src/methods/response.spec.ts +++ b/packages/data-schemas/src/methods/response.spec.ts @@ -1517,3 +1517,101 @@ it('retries a failed initial ID assignment without reopening its unused source s await storage.reserve(reservation({ conversationId: input.newConversationId })), ).toMatchObject({ ok: true }); })); + +it('stages one exact execution receipt independently of repeated projection revisions', () => + scoped(async () => { + await storage.reserve(reservation({ sourceKeyId: 'original-key' })); + const running = await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: '{}', + }); + if (!running.ok) throw new Error('Publication failed'); + for (let i = 0; i < 30; i++) { + const current = (await storage.read(owner, 'resp_one'))!; + expect( + ( + await storage.publish(owner, 'resp_one', current.revision, { + status: 'in_progress', + responseJSON: JSON.stringify({ progress: i }), + }) + ).ok, + ).toBe(true); + } + expect( + await storage.stageTerminal(owner, 'resp_one', running.record.revision, outcome), + ).toEqual({ ok: false, code: 'response_conflict' }); + const attempts = await Promise.all( + Array.from({ length: 12 }, () => + storage.stageTerminal(owner, 'resp_one', { execution: running.record }, outcome), + ), + ); + expect(attempts.every((result) => result.ok)).toBe(true); + const staged = (await storage.read(owner, 'resp_one'))!; + expect(staged.pendingTerminal).toEqual(outcome); + expect(staged.revision).toBe(running.record.revision + 31); + expect( + await storage.stageTerminal( + owner, + 'resp_one', + { execution: running.record }, + { + ...outcome, + ownerId: 'another-worker', + }, + ), + ).toEqual({ ok: false, code: 'response_conflict' }); + expect((await storage.finishTerminal(owner, 'resp_one', staged.revision, outcome)).ok).toBe( + true, + ); + })); + +it.each(['agentId', 'conversationId', 'sourceKeyId', 'generationCreatedAt'] as const)( + 'does not stage a retained execution with a mismatched %s', + (field) => + scoped(async () => { + await storage.reserve(reservation({ sourceKeyId: 'original-key' })); + const running = await storage.publish(owner, 'resp_one', 0, { + status: 'in_progress', + generationCreatedAt: 10, + responseJSON: '{}', + }); + if (!running.ok) throw new Error('Publication failed'); + const execution = { + ...running.record, + [field]: field === 'generationCreatedAt' ? 11 : 'other', + }; + expect(await storage.stageTerminal(owner, 'resp_one', { execution }, outcome)).toEqual({ + ok: false, + code: 'response_conflict', + }); + expect((await storage.read(owner, 'resp_one'))?.pendingTerminal).toBeUndefined(); + }), +); + +it('matches the complete reviewed and resumed execution binding rather than its projection', () => + scoped(async () => { + await pause(); + await storage.reserveResume(owner, 'resp_one', resume(), 100); + await storage.admitResume(owner, 'resp_one', 'intent-one', 10, 'provider-two', '{}', 101); + const retained = (await storage.read(owner, 'resp_one'))!; + const receipt = { ...outcome, providerExecutionId: 'provider-two' }; + for (const execution of [ + { ...retained, review: undefined }, + { ...retained, review: { ...retained.review!, checkpointId: 'different' } }, + { ...retained, resume: { ...retained.resume!, fingerprint: 'different' } }, + { ...retained, resume: { ...retained.resume!, state: 'reserved' as const } }, + ]) + expect(await storage.stageTerminal(owner, 'resp_one', { execution }, receipt)).toEqual({ + ok: false, + code: 'response_conflict', + }); + const current = await storage.publish(owner, 'resp_one', retained.revision, { + status: 'in_progress', + responseJSON: '{"projection":"new"}', + }); + expect(current.ok).toBe(true); + expect( + (await storage.stageTerminal(owner, 'resp_one', { execution: retained }, receipt)).ok, + ).toBe(true); + })); diff --git a/packages/data-schemas/src/methods/response.ts b/packages/data-schemas/src/methods/response.ts index d50bc783a3..670511053a 100644 --- a/packages/data-schemas/src/methods/response.ts +++ b/packages/data-schemas/src/methods/response.ts @@ -77,11 +77,71 @@ export type ResponseResumeResult = | { ok: true; record: StoredResponseRecord; replayed: boolean } | { ok: false; code: 'resume_conflict' }; +/** A drained execution's authority is independent of polling's projection revision. */ +export type ResponseTerminalGuard = + | number + | { + execution: Pick< + StoredResponseRecord, + 'agentId' | 'sourceKeyId' | 'conversationId' | 'generationCreatedAt' | 'review' | 'resume' + >; + }; + +function terminalExecutionFilter(guard: Exclude) { + const { execution } = guard; + const filter: Record = { + agentId: execution.agentId, + sourceKeyId: execution.sourceKeyId ?? { $exists: false }, + conversationId: execution.conversationId, + generationCreatedAt: execution.generationCreatedAt, + }; + const review = execution.review; + if (review) { + const fields: Record< + keyof ResponseReviewCheckpoint, + ResponseReviewCheckpoint[keyof ResponseReviewCheckpoint] + > = { + actionId: review.actionId, + providerExecutionId: review.providerExecutionId, + expiresAt: review.expiresAt, + partialResponseJSON: review.partialResponseJSON, + visibleOutputTokens: review.visibleOutputTokens, + pendingActionJSON: review.pendingActionJSON, + checkpointConfigJSON: review.checkpointConfigJSON, + checkpointId: review.checkpointId, + checkpointNs: review.checkpointNs, + }; + for (const [key, value] of Object.entries(fields)) + filter[`review.${key}`] = value ?? { $exists: false }; + } else filter.review = { $exists: false }; + const resume = execution.resume; + if (resume) { + const fields: Record< + keyof ResponseResumeIntent, + ResponseResumeIntent[keyof ResponseResumeIntent] + > = { + intentId: resume.intentId, + actionId: resume.actionId, + previousProviderExecutionId: resume.previousProviderExecutionId, + providerExecutionId: resume.providerExecutionId, + fingerprint: resume.fingerprint, + resolutionJSON: resume.resolutionJSON, + decisionJSON: resume.decisionJSON, + state: resume.state, + reservedAt: resume.reservedAt, + admittedAt: resume.admittedAt, + }; + for (const [key, value] of Object.entries(fields)) + filter[`resume.${key}`] = value ?? { $exists: false }; + } else filter.resume = { $exists: false }; + return filter; +} + export interface ResponseStorage { stageTerminal( owner: ResponseOwner, responseId: string, - revision: number, + guard: ResponseTerminalGuard, receipt: ResponseTerminalReceipt, ): Promise; finishTerminal( @@ -183,7 +243,13 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp $and: [activeExpirationFilter()], }); return { - async stageTerminal(owner, responseId, revision, receipt) { + async stageTerminal(owner, responseId, guard, receipt) { + if ( + typeof guard !== 'number' && + guard.execution.generationCreatedAt !== receipt.generationCreatedAt + ) + return { ok: false, code: 'response_conflict' }; + const execution = typeof guard === 'number' ? undefined : terminalExecutionFilter(guard); const pendingTerminal: ResponseTerminalReceipt = { generationCreatedAt: receipt.generationCreatedAt, providerExecutionId: receipt.providerExecutionId, @@ -202,7 +268,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp { ...scope(owner), responseId, - revision, + ...(execution ?? { revision: guard }), generationCreatedAt: receipt.generationCreatedAt, status: { $in: active }, pendingTerminal: { $exists: false }, @@ -223,7 +289,7 @@ export function createResponseStorage(mongoose: typeof import('mongoose')): Resp ) .lean(); if (record) return { ok: true, record }; - const existing = await model.findOne({ ...scope(owner), responseId }).lean(); + const existing = await model.findOne({ ...scope(owner), responseId, ...execution }).lean(); if ( existing?.generationCreatedAt === receipt.generationCreatedAt && existing.pendingTerminal &&