diff --git a/packages/api/src/agents/responses/background.ts b/packages/api/src/agents/responses/background.ts index 0618fcffc2..39dfc3ed21 100644 --- a/packages/api/src/agents/responses/background.ts +++ b/packages/api/src/agents/responses/background.ts @@ -153,7 +153,22 @@ export function createBackgroundResponseService(deps: BackgroundResponseDependen : { ok: false, status: 404, code: 'response_not_found' }; } + /** Null is absence of a readable journal, never an authorization denial or storage failure. */ + async function lookup( + owner: ResponseOwner, + responseId: string, + ): Promise { + const record = await deps.storage.read(owner, responseId); + if (!record) return null; + if (!(await deps.authorize(owner, record.agentId, record.sourceKeyId))) + return { ok: false, status: 404, code: 'response_not_found' }; + return responseResult(record); + } + return { + lookup(owner: ResponseOwner, responseId: string): Promise { + return lookup(owner, responseId); + }, async create( envelope: ResponsesRunEnvelope, idempotencyKey?: string, @@ -199,9 +214,13 @@ export function createBackgroundResponseService(deps: BackgroundResponseDependen : { ok: true, response: view(record) }; }, async retrieve(owner: ResponseOwner, responseId: string): Promise { - const record = await readAuthorized(owner, responseId); - if (!record) return { ok: false, status: 404, code: 'response_not_found' }; - return responseResult(record); + return ( + (await lookup(owner, responseId)) ?? { + ok: false, + status: 404, + code: 'response_not_found', + } + ); }, async cancel(owner: ResponseOwner, responseId: string): Promise { const authorized = await readAuthorized(owner, responseId); diff --git a/packages/api/src/agents/responses/foreground.spec.ts b/packages/api/src/agents/responses/foreground.spec.ts index d64b345b0a..3f4373bccc 100644 --- a/packages/api/src/agents/responses/foreground.spec.ts +++ b/packages/api/src/agents/responses/foreground.spec.ts @@ -134,6 +134,26 @@ function appFor( return app; } +it.each([conversationId, responseId])( + 'does not project background history through an ordinary %s read', + (id) => + scoped(async () => { + await save({ ...result(), background: true }); + await request(appFor()) + .get('/' + id) + .expect(404); + // A damaged full receipt cannot make its server-written background marker disappear. + await mongoose.models.Message.updateOne( + { user: owner.userId, messageId: responseId }, + { $set: { 'metadata.responseReceipt': { background: true } } }, + ); + const rejected = await request(appFor()) + .get('/' + id) + .expect(404); + expect(JSON.stringify(rejected.body)).not.toContain('tool-result-only-in-canonical-content'); + }), +); + it.each([conversationId, responseId])('preserves legacy tokenCount usage for %s', (id) => scoped(async () => { await db.saveConvo({ userId: owner.userId }, { conversationId, endpoint: 'agents' }); diff --git a/packages/api/src/agents/responses/host.spec.ts b/packages/api/src/agents/responses/host.spec.ts new file mode 100644 index 0000000000..863fe3af79 --- /dev/null +++ b/packages/api/src/agents/responses/host.spec.ts @@ -0,0 +1,1464 @@ +import express from 'express'; +import request from 'supertest'; +import mongoose from 'mongoose'; +import { networkInterfaces } from 'node:os'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { PermissionBits, FileSources, RetentionMode } from 'librechat-data-provider'; +import { + createModels, + createMethods, + createResponseStorage, + tenantStorage, + runAsSystem, + AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1, +} from '@librechat/data-schemas'; +import type { AppConfig, IUser, ResponseStorage } from '@librechat/data-schemas'; +import type { RequestHandler } from 'express'; +import type { Server } from 'node:http'; +import type { AgentTriggerDeliveryPersistence } from '../triggers/service'; +import type { AgentExecutionEnrollment } from '../remote/lifecycle'; +import type { BackgroundResponsesHostDependencies } from './host'; +import type { ApiKeyAuthRequest } from '~/apiKeys/middleware'; +import type { ServerRequest } from '~/types/http'; +import { + buildAggregatedResponse, + createResponseAggregator, + createResponseContext, + convertInputToMessages, +} from './service'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { createCheckpointNamespace } from '~/stream/checkpoints'; +import { createAgentTriggerService } from '../triggers/service'; +import { createRequireApiKeyAuth } from '~/apiKeys/middleware'; +import { buildPendingBackgroundResponse } from './background'; +import { mintBackgroundResponseWorkerToken } from './worker'; +import { enrollAgentExecution } from '../remote/lifecycle'; +import { createResponseExecutionRuntime } from './runtime'; +import { buildResponseInputChain } from './persistence'; +import { createBackgroundResponsesHost } from './host'; +import { getSafeErrorMetadata } from '~/utils/errors'; +import { createAgentRunEnvelope } from '../envelope'; +import { selfOriginFromAddress } from '~/app/origin'; +import { buildPendingAction } from '../hitl/policy'; +import { executeAgentRun } from '../remote/host'; +import { GenerationJobManager } from '~/stream'; +import { createResponseTurn } from './turn'; + +let mongo: MongoMemoryReplSet; +let jobStore: InMemoryJobStore; +let db: ReturnType; +let storage: ResponseStorage; +let user: IUser; +let tenantId: string | undefined = 'responses-host-test'; +const scoped = (work: () => Promise) => tenantStorage.run({ tenantId }, work); +const cleanup: Array<() => Promise> = []; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + db = createMethods(mongoose); + storage = createResponseStorage(mongoose); +}); +beforeEach(async () => { + tenantId = 'responses-host-test'; + await runAsSystem(async () => { + await Promise.all( + [ + 'StoredResponse', + 'ResponseScope', + 'AgentTriggerDelivery', + 'AgentTriggerLaneSequence', + 'AgentTriggerUserPurge', + 'AgentApiKey', + 'Conversation', + 'Message', + 'User', + ].map((name) => mongoose.models[name].deleteMany({})), + ); + }); + user = await scoped(async () => + mongoose.models.User.create({ + email: 'response-host@example.test', + name: 'Response test', + role: 'USER', + provider: 'local', + tenantId, + }), + ); + jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 }); + GenerationJobManager.configure({ + jobStore, + eventTransport: new InMemoryEventTransport(), + cleanupOnComplete: false, + }); + GenerationJobManager.initialize(); +}); +afterEach(async () => { + await GenerationJobManager.destroy(); + while (cleanup.length) await cleanup.pop()!(); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); + +function persistence(): AgentTriggerDeliveryPersistence { + return { + ...db, + getAgentTriggerDeliveryBatch: (delivery) => + db.getAgentTriggerDeliveryBatch({ + id: delivery.id, + batchMemberIds: delivery.batchMemberIds?.map( + (id) => new mongoose.Types.ObjectId(String(id)), + ), + }), + }; +} + +async function fixture( + decisions?: BackgroundResponsesHostDependencies['decisions'], + interactive?: BackgroundResponsesHostDependencies['interactive'], + listenAddress = '127.0.0.1', +) { + const config: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { + agents: { remoteApi: { background: { enabled: true, acknowledgementTimeoutMs: 1000 } } }, + }, + }; + const errors: unknown[] = []; + const inputs: unknown[] = []; + const ordinaryRetrievals: string[] = []; + let browserUser: IUser | undefined = user; + let browserMode: 'owner' | 'trigger' | 'key' = 'owner'; + let generate: ( + signal: AbortSignal, + context: { streamId: string | null; jobCreatedAt?: number }, + execution: AgentExecutionEnrollment, + ) => Promise = async () => '# Real Responses lifecycle'; + const host = createBackgroundResponsesHost({ + decisions, + interactive, + storage, + jobs: GenerationJobManager, + publicBaseURL: 'https://agents.example', + findUser: (id) => db.findUser({ _id: id }), + getKey: db.getAgentApiKeyById, + getRole: async () => + new mongoose.models.Role({ name: 'USER', permissions: { REMOTE_AGENTS: { USE: true } } }), + getConfig: async () => config, + isPrincipalActive: async () => true, + getAgentPermissions: async (agentId) => (agentId === 'agent' ? PermissionBits.VIEW : 0), + getMessage: db.getMessage, + getConvoRetention: db.getConvoRetention, + enqueue: (...args) => service.enqueue(...args), + signingSecret: () => 'isolated-background-host-secret', + onError: (error) => { + errors.push(error); + }, + }); + const apiKeyAuth = createRequireApiKeyAuth({ + validateAgentApiKey: db.validateAgentApiKey, + findUser: db.findUser, + isPrincipalActive: async () => true, + enrollmentRequired: () => false, + }); + const handlers = host.handlers({ + create: async (req, res) => { + const current = (req as ServerRequest).user!; + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: 'http-request', + receivedAt: Date.now(), + principal: { id: current.id, tenantId: current.tenantId }, + payload: req.body, + }); + const runtime = createResponseExecutionRuntime(req, res, { + getMessage: db.getMessage, + jobs: GenerationJobManager, + }); + const continuation = await runtime.resolveContinuation(envelope); + const turn = createResponseTurn({ + runtime, + context: createResponseContext(envelope.payload, runtime.responseId), + requestedStore: true, + logger: { error: (message, metadata) => errors.push({ message, metadata }) }, + }); + await executeAgentRun({ + envelope, + runId: runtime.responseId, + conversationId: continuation.conversationId, + connection: runtime.connection, + ...turn.hooks, + captureTerminalOutcome: true, + isPrincipalActive: async () => true, + execute: async (execution) => { + inputs.push(envelope.payload); + const text = await generate(execution.signal, runtime.toolContext(execution), execution); + if (typeof text !== 'string') return; + const aggregator = createResponseAggregator(); + aggregator.addText(text); + await turn.stage( + { + request: { + ...(req as ServerRequest), + body: { isTemporary: envelope.payload.isTemporary }, + }, + run: { getInterrupt: () => undefined, getRunMessages: () => [] }, + execution, + agents: [], + response: buildAggregatedResponse( + createResponseContext(envelope.payload, runtime.responseId), + aggregator, + ), + conversationId: continuation.conversationId, + agentId: envelope.payload.model, + agent: { id: envelope.payload.model, name: 'HTTP Test Agent' }, + ...runtime.inputChain( + buildResponseInputChain( + convertInputToMessages(envelope.payload.input), + continuation.parentMessageId, + ), + ), + transport: { type: 'json', response: res }, + }, + { + saveConvo: db.saveConvo, + saveMessage: async (...args) => { + const saved = await db.saveMessage(...args); + return saved + ? { conversationId: saved.conversationId, messageId: saved.messageId } + : null; + }, + announceReply: async () => {}, + resolveTitle: () => null, + traceFields: async () => ({ langfuseSampled: false }), + }, + ); + }, + handleExecutionError: () => { + if (!res.writableEnded) res.status(409).json({ error: { code: 'execution_failed' } }); + }, + onSettlementError: (error) => { + errors.push(error); + }, + }); + }, + retrieve: (req, res) => { + ordinaryRetrievals.push(req.params.id); + res.status(404).json({ error: { code: 'ordinary_response_not_found' } }); + }, + }); + const app = express(); + app.use(express.json()); + const browserAuth: RequestHandler = (req, _res, next) => { + Object.assign(req, { + user: browserUser, + _isAgentTrigger: browserMode === 'trigger', + apiKeyId: browserMode === 'key' ? 'execution-key' : undefined, + }); + next(); + }; + app.get('/api/agents/responses/:id/review', browserAuth, host.review); + app.post('/api/agents/responses/:id/review', browserAuth, host.submitReview); + app.post('/api/agents/responses/:id/cancel', browserAuth, host.cancelReview); + app.use((_req, _res, next) => { + tenantStorage.run({ tenantId }, async () => { + next(); + }); + }); + const router = express.Router(); + router.use( + host.authenticate(async (req, res, next) => { + await apiKeyAuth(req as ApiKeyAuthRequest, res, next); + }), + ); + router.post('/', handlers.create); + router.get('/:id', handlers.retrieve); + router.post('/:id/cancel', handlers.cancel); + app.use('/api/agents/v1/responses', router); + const server: Server = await new Promise((resolve) => { + const listener = app.listen(0, listenAddress, () => resolve(listener)); + }); + cleanup.push(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + host.initialize(server.address()); + const service = createAgentTriggerService({ + methods: persistence(), + sourceDispatchers: host.sourceDispatchers, + deliveryOptions: { tickMs: 10 }, + }); + await service.initialize({ address: server.address() }); + cleanup.push(() => service.stop()); + const credential = await scoped(async () => + db.createAgentApiKey({ userId: user._id, name: 'background-test' }), + ); + const origin = selfOriginFromAddress(server.address()); + if (!origin) throw new Error('Missing initialized HTTP listener'); + const client = request(origin); + async function readUntil(id: string, status: string) { + let observed: { httpStatus: number; status?: string; errorCode?: string } | undefined; + const failure = () => + new Error( + `Response did not reach ${status}: ${JSON.stringify({ + observed, + executions: inputs.length, + workerErrors: errors.map(getSafeErrorMetadata), + })}`, + ); + for (let attempt = 0; attempt < 150; attempt++) { + const response = await client + .get(`/api/agents/v1/responses/${id}`) + .auth(credential.key, { type: 'bearer' }); + observed = { + httpStatus: response.status, + status: response.body.status, + errorCode: response.body.error?.code, + }; + if (response.body.status === status) return response; + if (['completed', 'failed', 'cancelled', 'incomplete'].includes(response.body.status)) { + throw failure(); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw failure(); + } + return { + host, + client, + credential, + inputs, + config, + errors, + ordinaryRetrievals, + readUntil, + setBrowserUser: (value?: IUser) => { + browserUser = value; + }, + setBrowserMode: (value: typeof browserMode) => { + browserMode = value; + }, + setGenerate: (value: typeof generate) => { + generate = value; + }, + }; +} + +async function ownerPause(keyId: string) { + return scoped(async () => { + const owner = { userId: String(user._id), tenantId }; + const id = 'resp_owner_review'; + const execution = await enrollAgentExecution({ + runId: id, + ...owner, + agentId: 'agent', + conversationId: 'review-conversation', + protocol: 'responses', + isPrincipalActive: async () => true, + }); + const reserved = await storage.reserve({ + ...owner, + responseId: id, + sourceKeyId: keyId, + agentId: 'agent', + requestKey: id, + fingerprint: id, + conversationId: 'review-conversation', + parentMessageId: 'root', + requestJSON: '{"model":"agent","input":"Question","background":true,"store":true}', + }); + if (!reserved.ok) throw new Error('Review reservation failed'); + const body = JSON.stringify({ + ...buildPendingBackgroundResponse(reserved.record), + status: 'in_progress', + }); + await storage.publish(owner, id, 0, { + status: 'in_progress', + generationCreatedAt: execution.createdAt, + responseJSON: body, + }); + await execution.beginProviderExecution(); + const job = (await GenerationJobManager.getJob(id))!; + const action = buildPendingAction( + { type: 'ask_user_question', question: { question: 'Choose an environment' } }, + { + streamId: id, + responseMessageId: id, + conversationId: 'review-conversation', + threadId: 'review-conversation', + ttlMs: 60000, + }, + ); + expect( + await execution.pauseForReview(action, async () => { + const published = await storage.publishReview( + owner, + id, + 1, + execution.createdAt, + { + actionId: action.actionId, + providerExecutionId: job.metadata.providerExecutionId!, + expiresAt: action.expiresAt!, + partialResponseJSON: body, + pendingActionJSON: JSON.stringify(action), + }, + body, + ); + if (!published.ok) throw new Error('Review publication failed'); + }), + ).toBe(true); + await execution.settle(); + return { id, action, owner }; + }); +} + +it('requires owner browser authentication and leaves submission closed without an installed validator/queue', async () => { + const f = await fixture(); + const path = '/api/agents/responses/anything/review'; + const closed = await f.client + .post(path) + .send({ actionId: 'action', answer: 'staging' }) + .expect(503); + expect(closed.body).toEqual({ error: 'response_decision_unavailable' }); + expect(closed.headers['cache-control']).toBe('no-store'); + f.setBrowserUser(undefined); + await f.client.post(path).send({}).expect(401); + f.setBrowserUser(user); + for (const mode of ['trigger', 'key'] as const) { + f.setBrowserMode(mode); + await f.client.post(path).send({}).expect(401); + await f.client.get(path).expect(401); + } +}); + +it('submits an owner decision through HTTP once, rechecks the source key, and never executes from the control route', async () => { + const queued: string[] = []; + const validate = jest.fn(async () => true); + const f = await fixture({ + validate, + enqueue: async (record) => { + queued.push(record.resume!.intentId); + }, + }); + const { id, action, owner } = await ownerPause(f.credential.id); + const path = `/api/agents/responses/${id}/review`; + for (const other of [ + new mongoose.models.User({ ...user.toObject(), _id: new mongoose.Types.ObjectId() }), + new mongoose.models.User({ ...user.toObject(), tenantId: 'another-tenant' }), + ]) { + f.setBrowserUser(other); + await f.client.post(path).send({ actionId: action.actionId, answer: 'staging' }).expect(404); + } + expect(validate).not.toHaveBeenCalled(); + f.setBrowserUser(user); + const first = await f.client + .post(path) + .send({ actionId: action.actionId, answer: 'staging' }) + .expect(202); + const again = await f.client + .post(path) + .send({ answer: 'staging', actionId: action.actionId }) + .expect(202); + expect(again.body).toEqual(first.body); + expect(first.body).toMatchObject({ responseId: id, actionId: action.actionId, status: 'queued' }); + expect(JSON.stringify(first.body)).not.toContain('staging'); + expect(new Set(queued)).toEqual(new Set([first.body.intentId])); + expect((await GenerationJobManager.getJob(id))?.status).toBe('requires_action'); + expect(f.inputs).toEqual([]); + await f.client + .post(path) + .send({ actionId: action.actionId, answer: 'staging', userId: 'another-owner' }) + .expect(400); + await scoped(() => db.deleteAgentApiKey(f.credential.id, user._id)); + await f.client.post(path).send({ actionId: action.actionId, answer: 'staging' }).expect(404); + expect(queued).toHaveLength(2); + expect((await scoped(() => storage.read(owner, id)))?.resume?.intentId).toBe(first.body.intentId); +}); + +it('denies owner review and first decision after the paused original source key is revoked', async () => { + const validate = jest.fn(async () => true); + const enqueue = jest.fn(async () => {}); + const f = await fixture({ validate, enqueue }); + const { id, action, owner } = await ownerPause(f.credential.id); + const path = `/api/agents/responses/${id}/review`; + await scoped(() => db.deleteAgentApiKey(f.credential.id, user._id)); + + await f.client.get(path).expect(404); + await f.client.post(path).send({ actionId: action.actionId, answer: 'staging' }).expect(404); + + expect(validate).not.toHaveBeenCalled(); + expect(enqueue).not.toHaveBeenCalled(); + expect(f.inputs).toEqual([]); + const retained = await scoped(() => storage.read(owner, id)); + expect(retained?.status).toBe('requires_action'); + expect(retained?.resume).toBeUndefined(); + expect((await GenerationJobManager.getJob(id))?.status).toBe('requires_action'); +}); + +it('reuses canonical decision receipts and source preflight for a trusted native transport without browser tokens', async () => { + const queued: string[] = []; + const f = await fixture({ + validate: async () => true, + enqueue: async (record) => { + queued.push(record.resume!.intentId); + }, + }); + const { id, action, owner } = await ownerPause(f.credential.id); + const view = await scoped(() => f.host.readInteractiveReview(owner, id)); + expect(view?.pendingAction?.actionId).toBe(action.actionId); + expect(view?.oauth).toEqual([]); + for (const other of [ + { ...owner, userId: 'another-user' }, + { ...owner, tenantId: 'another-tenant' }, + ]) { + expect(await f.host.readInteractiveReview(other, id)).toBeNull(); + expect( + await f.host.submitInteractiveReview(other, id, { actionId: action.actionId, answer: 'yes' }), + ).toMatchObject({ ok: false, status: 404 }); + } + const input = { actionId: action.actionId, answer: 'yes' }; + const first = await scoped(() => f.host.submitInteractiveReview(owner, id, input)); + expect(first).toMatchObject({ ok: true, receipt: { responseId: id, actionId: action.actionId } }); + expect(await scoped(() => f.host.submitInteractiveReview(owner, id, input))).toEqual(first); + expect( + await scoped(() => f.host.submitInteractiveReview(owner, id, { ...input, answer: 'changed' })), + ).toMatchObject({ ok: false, status: 409 }); + expect(new Set(queued).size).toBe(1); + expect(f.inputs).toEqual([]); + await scoped(() => db.deleteAgentApiKey(f.credential.id, user._id)); + expect(await scoped(() => f.host.readInteractiveReview(owner, id))).toBeNull(); + expect(await scoped(() => f.host.submitInteractiveReview(owner, id, input))).toMatchObject({ + ok: false, + status: 404, + }); +}); + +it('registers only resume-capable initial/resume workers and dispatches saved intent through signed Responses ingress', async () => { + const admitResume = jest.fn< + ReturnType['admitResume']>, + Parameters['admitResume']> + >(async () => ({ ok: false, code: 'response_resume_unavailable' })); + const validate = jest.fn(async () => true); + const f = await fixture(undefined, { validate, admitResume }); + expect([...f.host.sourceDispatchers.keys()].sort()).toEqual(['responses', 'responses.resume']); + for (const dispatcher of f.host.sourceDispatchers.values()) { + expect(dispatcher.workerCapability).toBe(AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1); + } + const { id, action, owner } = await ownerPause(f.credential.id); + const receipt = await f.client + .post(`/api/agents/responses/${id}/review`) + .send({ actionId: action.actionId, answer: 'staging' }) + .expect(202); + for (let attempt = 0; attempt < 100 && !admitResume.mock.calls.length; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(validate).toHaveBeenCalledTimes(1); + expect(admitResume).toHaveBeenCalled(); + const [record, authenticatedUser, intentId] = admitResume.mock.calls[0]; + expect(record.responseId).toBe(id); + expect(record.resume?.decisionJSON).toBe( + JSON.stringify({ actionId: action.actionId, answer: 'staging' }), + ); + expect(String(authenticatedUser._id)).toBe(owner.userId); + expect(intentId).toBe(receipt.body.intentId); + expect(f.inputs).toEqual([]); + expect((await GenerationJobManager.getJob(id))?.status).toBe('requires_action'); +}); + +it('returns a safe retryable HTTP failure when queue publication fails after saving the owner decision', async () => { + let calls = 0; + const f = await fixture({ + validate: async () => true, + enqueue: async () => { + if (++calls === 1) throw new Error('private queue diagnostics'); + }, + }); + const { id, action, owner } = await ownerPause(f.credential.id); + const path = `/api/agents/responses/${id}/review`; + const failed = await f.client + .post(path) + .send({ actionId: action.actionId, answer: 'staging' }) + .expect(503); + expect(failed.body).toEqual({ error: 'response_service_unavailable' }); + expect(JSON.stringify(failed.body)).not.toContain('private queue'); + const restored = await f.client.get(path).expect(200); + expect(restored.body.canSubmitDecision).toBe(true); + expect(restored.body.pendingDecision.decision).toEqual({ + actionId: action.actionId, + answer: 'staging', + }); + const native = await scoped(() => f.host.readInteractiveReview(owner, id)); + expect(native?.pendingDecision).toEqual(restored.body.pendingDecision); + expect(native?.pendingAction?.actionId).toBe(action.actionId); + const retry = await f.client.post(path).send(restored.body.pendingDecision.decision).expect(202); + expect(retry.body.intentId).toBe(restored.body.pendingDecision.intentId); + expect(calls).toBe(2); + expect(f.errors).toHaveLength(1); +}); + +it('does not publish a completed response while its provider has durably paused for review', async () => { + const f = await fixture(); + f.setGenerate(async (_signal, _context, execution) => { + const job = await GenerationJobManager.getJob(execution.runId); + if (!job?.metadata.conversationId) throw new Error('Missing admitted conversation'); + const action = buildPendingAction( + { type: 'ask_user_question', question: { question: 'Which environment?' } }, + { + streamId: execution.runId, + responseMessageId: execution.runId, + runId: execution.runId, + conversationId: job.metadata.conversationId, + threadId: job.metadata.conversationId, + ttlMs: 60_000, + }, + ); + const checkpointNamespace = createCheckpointNamespace(String(user._id), tenantId); + await GenerationJobManager.getJobStore().updateJob(execution.runId, { checkpointNamespace }); + if ( + !(await execution.pauseForReview(action, async () => { + const record = (await storage.read( + { userId: String(user._id), tenantId }, + execution.runId, + ))!; + const published = await storage.publishReview( + record, + record.responseId, + record.revision, + execution.createdAt, + { + actionId: action.actionId, + providerExecutionId: job.metadata.providerExecutionId!, + expiresAt: action.expiresAt!, + pendingActionJSON: JSON.stringify(action), + partialResponseJSON: '{}', + checkpointConfigJSON: JSON.stringify({ + type: 'mongo', + checkpointCollectionName: 'responses_pause_checkpoints', + checkpointWritesCollectionName: 'responses_pause_checkpoint_writes', + }), + }, + JSON.stringify({ + id: record.responseId, + object: 'response', + model: 'agent', + status: 'in_progress', + background: true, + store: true, + output: [], + completed_at: null, + }), + ); + if (!published.ok) throw new Error('Pause journal publication failed'); + for (const name of ['responses_pause_checkpoints', 'responses_pause_checkpoint_writes']) { + await mongoose.connection.db!.collection(name).insertMany([ + { + thread_id: record.conversationId, + checkpoint_ns: checkpointNamespace, + checkpoint_id: 'paused-root', + }, + { + thread_id: record.conversationId, + checkpoint_ns: `${checkpointNamespace}|child`, + checkpoint_id: 'paused-child', + }, + { + thread_id: record.conversationId, + checkpoint_ns: 'other-generation', + checkpoint_id: 'keep-me', + }, + ]); + } + })) + ) + throw new Error('Pause admission failed'); + return { paused: true }; + }); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Ask a question', background: true, store: true }) + .expect(202); + let job = await GenerationJobManager.getJob(created.body.id); + for ( + let attempt = 0; + attempt < 150 && + (job?.status !== 'requires_action' || + job.metadata.terminalPersistencePending !== false || + job.metadata.providerDrained !== true); + attempt++ + ) { + await new Promise((resolve) => setTimeout(resolve, 10)); + job = await GenerationJobManager.getJob(created.body.id); + } + expect(job).toMatchObject({ + status: 'requires_action', + metadata: { providerDrained: true, terminalPersistencePending: false }, + }); + const observed = await f.client + .get(`/api/agents/v1/responses/${created.body.id}`) + .auth(f.credential.key, { type: 'bearer' }) + .expect(200); + expect(observed.body).toMatchObject({ + status: 'in_progress', + output: [], + completed_at: null, + required_action: { type: 'librechat_review', kind: 'question' }, + }); + const paused = await scoped(() => + storage.read({ userId: String(user._id), tenantId }, created.body.id), + ); + expect(paused?.preparedTerminal).toBeUndefined(); + expect(paused?.pendingTerminal).toBeUndefined(); + expect(f.inputs).toHaveLength(1); + expect(f.errors).toEqual([]); + const cancelled = await f.client + .post(`/api/agents/v1/responses/${created.body.id}/cancel`) + .auth(f.credential.key, { type: 'bearer' }) + .send({}) + .expect(200); + expect(cancelled.body).toMatchObject({ status: 'cancelled', output: [] }); + for (const name of ['responses_pause_checkpoints', 'responses_pause_checkpoint_writes']) { + const rows = await mongoose.connection.db!.collection(name).find({}).toArray(); + expect(rows).toHaveLength(1); + expect(rows[0].checkpoint_id).toBe('keep-me'); + } + expect(f.errors).toEqual([]); +}); + +it('serves OAuth secrets only to the authenticated current owner, never the API key caller', async () => { + const f = await fixture(); + let finish: (value: string) => void = () => {}; + f.setGenerate(async (_signal, context) => { + expect(context.streamId).toMatch(/^resp_/); + expect(context.jobCreatedAt).toEqual(expect.any(Number)); + await GenerationJobManager.emitChunk( + context.streamId!, + { + event: 'on_run_step', + data: { + id: 'oauth-step', + runId: 'oauth-run', + index: 0, + stepDetails: { + type: 'tool_calls', + tool_calls: [{ id: 'oauth-call', name: 'oauth_mcp_example', args: '' }], + }, + }, + }, + { expectedCreatedAt: context.jobCreatedAt }, + ); + await GenerationJobManager.emitChunk( + context.streamId!, + { + event: 'on_run_step_delta', + data: { + id: 'oauth-step', + delta: { + type: 'tool_calls', + tool_calls: [{ name: 'oauth_mcp_example', args: 'private-args' }], + auth: 'https://oauth.example/authorize?state=private-state', + expires_at: Date.now() + 60_000, + }, + }, + }, + { expectedCreatedAt: context.jobCreatedAt }, + ); + return new Promise((resolve) => { + finish = resolve; + }); + }); + try { + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Use a tool', background: true, store: true }); + const id = created.body.id; + let observed = await f.readUntil(id, 'in_progress'); + for (let n = 0; n < 100 && !observed.body.required_action; n++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + observed = await f.client + .get(`/api/agents/v1/responses/${id}`) + .auth(f.credential.key, { type: 'bearer' }); + } + expect(observed.body.required_action).toMatchObject({ + kind: 'oauth', + review_url: `https://agents.example/responses/${id}/review`, + }); + expect(JSON.stringify(observed.body)).not.toMatch(/private-|oauth.example|oauth_mcp_example/); + const review = await f.client.get(`/api/agents/responses/${id}/review`); + expect(review.status).toBe(200); + expect(review.headers['cache-control']).toBe('no-store'); + expect(review.body.oauth).toEqual([ + expect.objectContaining({ authURL: 'https://oauth.example/authorize?state=private-state' }), + ]); + f.setBrowserUser(undefined); + expect( + ( + await f.client + .get(`/api/agents/responses/${id}/review`) + .auth(f.credential.key, { type: 'bearer' }) + ).status, + ).toBe(401); + f.setBrowserUser( + new mongoose.models.User({ ...user.toObject(), _id: new mongoose.Types.ObjectId() }), + ); + expect((await f.client.get(`/api/agents/responses/${id}/review`)).status).toBe(404); + f.setBrowserUser(new mongoose.models.User({ ...user.toObject(), tenantId: 'another-tenant' })); + expect((await f.client.get(`/api/agents/responses/${id}/review`)).status).toBe(404); + f.setBrowserUser(user); + await scoped(() => db.deleteAgentApiKey(f.credential.id, String(user._id))); + expect((await f.client.get(`/api/agents/responses/${id}/review`)).status).toBe(404); + } finally { + finish('Authorization finished'); + } +}); + +it('creates, executes and retrieves a background response through HTTP, real auth and the durable queue', async () => { + const f = await fixture(); + const body = { + model: 'agent', + input: 'Keep original input', + instructions: 'Keep these instructions', + background: true, + store: true, + max_output_tokens: 100, + metadata: { purpose: 'test' }, + }; + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .set('Idempotency-Key', 'same-event') + .send(body); + expect(created.status).toBe(202); + const result = await f.readUntil(created.body.id, 'completed'); + expect(result.body.output[0].content[0].text).toBe('# Real Responses lifecycle'); + expect(f.inputs).toEqual([body]); + const replay = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .set('Idempotency-Key', 'same-event') + .send(body); + expect(replay.body.id).toBe(created.body.id); + expect(f.inputs).toHaveLength(1); + const reply = await scoped(() => + db.getMessage({ user: String(user._id), messageId: created.body.id }), + ); + expect(reply).toMatchObject({ finish_reason: 'stop', text: '# Real Responses lifecycle' }); + const messages = await scoped(() => + db.getMessages({ user: String(user._id), conversationId: reply!.conversationId }), + ); + expect(messages).toHaveLength(2); + expect(messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + messageId: reply!.parentMessageId, + isCreatedByUser: true, + text: body.input, + }), + ]), + ); + expect(f.errors).toEqual([]); +}); + +it.each([RetentionMode.ALL, RetentionMode.EPHEMERAL])( + 'captures %s retention before storing a new private background request', + async (retentionMode) => { + const retention = jest.spyOn(db, 'getConvoRetention'); + cleanup.push(async () => retention.mockRestore()); + const f = await fixture(); + f.config.interfaceConfig = { + retentionMode, + generalChatRetention: 2, + temporaryChatRetention: 1, + }; + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'retained private', background: true }); + expect(created.status).toBe(202); + expect(retention).not.toHaveBeenCalled(); + const row = await scoped(() => + storage.read({ userId: String(user._id), tenantId }, created.body.id), + ); + expect(row?.expiredAt).toBeInstanceOf(Date); + const hours = retentionMode === RetentionMode.ALL ? 2 : 1; + expect(row!.expiredAt!.getTime() - row!.createdAt.getTime()).toBeGreaterThan( + hours * 3_600_000 - 5000, + ); + expect(row!.expiredAt!.getTime() - row!.createdAt.getTime()).toBeLessThanOrEqual( + hours * 3_600_000, + ); + expect(row?.isTemporary).toBe(retentionMode === RetentionMode.EPHEMERAL); + await f.readUntil(created.body.id, 'completed'); + expect(f.errors).toEqual([]); + }, +); + +it('inherits the existing temporary parent deadline instead of renewing its retention window', async () => { + const retention = jest.spyOn(db, 'getConvoRetention'); + cleanup.push(async () => retention.mockRestore()); + const f = await fixture(); + f.config.interfaceConfig = { + retentionMode: RetentionMode.ALL, + generalChatRetention: 2, + temporaryChatRetention: 1, + }; + const conversationId = '00000000-0000-4000-8000-000000000001'; + const expiredAt = new Date(Date.now() + 300_000); + await scoped(() => + mongoose.models.Conversation.create({ + user: String(user._id), + conversationId, + expiredAt, + isTemporary: true, + endpoint: 'agents', + }), + ); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ + model: 'agent', + input: 'continued', + background: true, + previous_response_id: conversationId, + }); + expect(created.status).toBe(202); + expect(retention).toHaveBeenCalledTimes(1); + expect(retention).toHaveBeenCalledWith(String(user._id), conversationId); + expect( + await scoped(() => storage.read({ userId: String(user._id), tenantId }, created.body.id)), + ).toMatchObject({ expiredAt, isTemporary: true, conversationId }); + await f.readUntil(created.body.id, 'completed'); + expect(f.errors).toEqual([]); +}); + +it('rejects an expired or missing continuation before writing its private request', async () => { + const f = await fixture(); + const conversationId = 'expired-response-parent'; + await scoped(() => + mongoose.models.Conversation.create({ + user: String(user._id), + conversationId, + expiredAt: new Date(Date.now() - 1), + isTemporary: true, + endpoint: 'agents', + }), + ); + const expired = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ + model: 'agent', + input: 'must not persist', + background: true, + previous_response_id: conversationId, + }); + expect(expired.status).toBe(409); + const missing = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ + model: 'agent', + input: 'also must not persist', + background: true, + previous_response_id: 'missing-parent', + }); + expect(missing.status).toBeGreaterThanOrEqual(400); + expect(await scoped(() => mongoose.models.StoredResponse.countDocuments({}))).toBe(0); + expect(f.inputs).toHaveLength(0); +}); + +it('keeps expired output private and releases a drained provider even after TTL removes its row', async () => { + const f = await fixture(); + let started!: () => void; + let finish!: () => void; + const running = new Promise((resolve) => { + started = resolve; + }); + const gate = new Promise((resolve) => { + finish = resolve; + }); + f.setGenerate(async () => { + started(); + await gate; + return 'late private output'; + }); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'private', background: true }); + await running; + const owner = { userId: String(user._id), tenantId }; + const record = (await scoped(() => storage.read(owner, created.body.id)))!; + await scoped(() => + db.saveConvo( + { userId: owner.userId, expiredAt: new Date(Date.now() - 1) }, + { + conversationId: record.conversationId, + endpoint: 'agents', + }, + ), + ); + expect( + ( + await f.client + .get(`/api/agents/v1/responses/${record.responseId}`) + .auth(f.credential.key, { type: 'bearer' }) + ).status, + ).toBe(404); + await scoped(() => mongoose.models.StoredResponse.deleteOne({ responseId: record.responseId })); + finish(); + for (let attempt = 0; attempt < 100; attempt++) { + const job = await GenerationJobManager.getJob(record.responseId); + if (job?.metadata.providerDrained && job.metadata.terminalPersistencePending !== true) break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(await GenerationJobManager.getJob(record.responseId)).toMatchObject({ + metadata: { + providerDrained: true, + terminalPersistencePending: false, + }, + }); + expect( + await scoped(() => + mongoose.models.StoredResponse.countDocuments({ responseId: record.responseId }), + ), + ).toBe(0); + expect(f.errors).toEqual([]); +}); + +it('does not start a provider when expiry wins enrollment publication and releases that job', async () => { + const f = await fixture(); + const publish = storage.publish.bind(storage); + const interrupted = jest.spyOn(storage, 'publish').mockImplementationOnce(async (...args) => { + const record = (await storage.read(args[0], args[1]))!; + await db.saveConvo( + { userId: record.userId, expiredAt: new Date(Date.now() - 1) }, + { + conversationId: record.conversationId, + endpoint: 'agents', + }, + ); + return publish(...args); + }); + try { + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'expired before start', background: true }); + for (let attempt = 0; attempt < 100; attempt++) { + const job = await GenerationJobManager.getJob(created.body.id); + if (job?.metadata.providerDrained && job.metadata.terminalPersistencePending === false) break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(f.inputs).toHaveLength(0); + expect(await GenerationJobManager.getJob(created.body.id)).toMatchObject({ + metadata: { + providerDrained: true, + terminalPersistencePending: false, + }, + }); + expect( + await scoped(() => storage.read({ userId: String(user._id), tenantId }, created.body.id)), + ).toBeNull(); + } finally { + interrupted.mockRestore(); + } +}); + +it('publishes terminal output when a concurrent read updates the review projection', async () => { + const f = await fixture(); + const stageTerminal = storage.stageTerminal.bind(storage); + const stage = jest.spyOn(storage, 'stageTerminal').mockImplementationOnce(async (...args) => { + const observed = await f.client + .get(`/api/agents/v1/responses/${args[1]}`) + .auth(f.credential.key, { type: 'bearer' }); + expect(observed.body).toMatchObject({ + status: 'in_progress', + recovery: 'execution_state_unavailable', + }); + return stageTerminal(...args); + }); + try { + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Finish despite polling', background: true, store: true }); + expect(created.status).toBe(202); + const result = await f.readUntil(created.body.id, 'completed'); + expect(result.body.output[0].content[0].text).toBe('# Real Responses lifecycle'); + expect(f.inputs).toHaveLength(1); + expect(f.errors).toEqual([]); + } finally { + stage.mockRestore(); + } +}); + +it('cancels the exact running response and waits for actual execution settlement', async () => { + const f = await fixture(); + f.setGenerate( + async (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('cancelled')), { once: true }); + }), + ); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Wait for Stop', background: true, store: true }); + await f.readUntil(created.body.id, 'in_progress'); + const cancelled = await f.client + .post(`/api/agents/v1/responses/${created.body.id}/cancel`) + .auth(f.credential.key, { type: 'bearer' }) + .send({}); + expect(cancelled.status).toBe(200); + await f.readUntil(created.body.id, 'cancelled'); + expect(f.inputs).toHaveLength(1); +}); + +it('lets only the browser owner stop the exact response after provider settlement', async () => { + GenerationJobManager.configure({ + jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }), + eventTransport: new InMemoryEventTransport(), + }); + const f = await fixture(); + let started!: () => void; + const providerStarted = new Promise((resolve) => { + started = resolve; + }); + let release!: () => void; + const drained = new Promise((resolve) => { + release = resolve; + }); + f.setGenerate(async (signal) => { + started(); + await new Promise((resolve) => { + signal.addEventListener('abort', () => resolve(), { once: true }); + }); + await drained; + throw new Error('cancelled'); + }); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Owner Stop', background: true, store: true }); + await providerStarted; + const id = created.body.id; + const exact = await GenerationJobManager.getJob(id); + let settled = false; + const stopping = f.client + .post(`/api/agents/responses/${id}/cancel`) + .send({}) + .then((result) => { + settled = true; + return result; + }); + try { + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); + expect((await scoped(() => storage.read({ userId: user.id, tenantId }, id)))?.status).toBe( + 'in_progress', + ); + expect(await GenerationJobManager.getJob(id)).toMatchObject({ + createdAt: exact!.createdAt, + metadata: { + providerExecutionId: exact!.metadata.providerExecutionId, + providerDrained: false, + }, + }); + } finally { + release(); + } + const stopped = await stopping; + expect(stopped.status).toBe(200); + expect(stopped.headers['cache-control']).toBe('no-store'); + expect(stopped.body).toMatchObject({ id, status: 'cancelled' }); + expect(await GenerationJobManager.getJob(id)).toBeUndefined(); + expect(f.inputs).toHaveLength(1); + await f.client.post(`/api/agents/responses/${id}/cancel`).send({}).expect(200); +}); + +it('denies browser cancellation for foreign owners, tenants, execution credentials, disabled policy and revoked source', async () => { + const f = await fixture(); + const { id, owner } = await ownerPause(f.credential.id); + const path = `/api/agents/responses/${id}/cancel`; + for (const other of [ + new mongoose.models.User({ ...user.toObject(), _id: new mongoose.Types.ObjectId() }), + new mongoose.models.User({ ...user.toObject(), tenantId: 'another-tenant' }), + ]) { + f.setBrowserUser(other); + await f.client.post(path).send({}).expect(404); + } + f.setBrowserUser(undefined); + await f.client.post(path).send({}).expect(401); + f.setBrowserUser(user); + for (const mode of ['trigger', 'key'] as const) { + f.setBrowserMode(mode); + await f.client.post(path).send({}).expect(401); + } + f.setBrowserMode('owner'); + f.config.endpoints!.agents!.remoteApi!.background!.enabled = false; + await f.client.post(path).send({}).expect(404); + f.config.endpoints!.agents!.remoteApi!.background!.enabled = true; + await scoped(() => db.deleteAgentApiKey(f.credential.id, user._id)); + await f.client.post(path).send({}).expect(404); + expect((await scoped(() => storage.read(owner, id)))?.cancellationRequestedAt).toBeUndefined(); + expect((await GenerationJobManager.getJob(id))?.status).toBe('requires_action'); + expect(f.inputs).toEqual([]); +}); + +it('does not let a new valid key resurrect an original revoked credential grant', async () => { + const f = await fixture(); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Hello', background: true, store: true }); + await f.readUntil(created.body.id, 'completed'); + const replacement = await scoped(async () => + db.createAgentApiKey({ userId: user._id, name: 'replacement-test' }), + ); + await scoped(async () => db.deleteAgentApiKey(f.credential.id, String(user._id))); + const result = await f.client + .get(`/api/agents/v1/responses/${created.body.id}`) + .auth(replacement.key, { type: 'bearer' }); + expect(result.status).toBe(404); + expect(result.body.error.code).toBe('response_not_found'); + expect(f.ordinaryRetrievals).toEqual([]); +}); + +it('preserves ordinary synchronous Responses and defaults background execution off when disabled', async () => { + const f = await fixture(); + f.config.endpoints!.agents!.remoteApi!.background!.enabled = false; + const ordinary = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Ordinary request' }); + expect(ordinary.status).toBe(200); + expect(ordinary.body.status).toBe('completed'); + const denied = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Background request', background: true, store: true }); + expect(denied.status).toBe(403); + expect(f.inputs).toHaveLength(1); +}); + +async function reserveUnstarted(credentialId: string) { + const result = await scoped(() => + storage.reserve({ + userId: String(user._id), + tenantId, + sourceKeyId: credentialId, + responseId: 'resp_delayed_worker', + requestKey: 'delayed-worker', + fingerprint: 'request', + agentId: 'agent', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'Delayed delivery', + background: true, + store: true, + }), + conversationId: 'delayed-conversation', + parentMessageId: 'root', + }), + ); + if (!result.ok) throw new Error('Reservation failed'); + return result.record; +} + +it('loads the journal once per poll and uses ordinary retrieval only for absent readable records', async () => { + const f = await fixture(); + const record = await reserveUnstarted(f.credential.id); + const read = jest.spyOn(storage, 'read'); + try { + await f.client + .get(`/api/agents/v1/responses/${record.responseId}`) + .auth(f.credential.key, { type: 'bearer' }) + .expect(200); + expect(read).toHaveBeenCalledTimes(1); + expect(f.ordinaryRetrievals).toEqual([]); + read.mockClear(); + const ordinary = await f.client + .get('/api/agents/v1/responses/resp_ordinary_fixture') + .auth(f.credential.key, { type: 'bearer' }) + .expect(404); + expect(ordinary.body.error.code).toBe('ordinary_response_not_found'); + expect(read).toHaveBeenCalledTimes(1); + expect(f.ordinaryRetrievals).toEqual(['resp_ordinary_fixture']); + } finally { + read.mockRestore(); + } +}); + +it('keeps journal outages out of the ordinary fallback and returns a safe retryable error', async () => { + const f = await fixture(); + const read = jest + .spyOn(storage, 'read') + .mockRejectedValueOnce(new Error('private journal query')); + try { + const result = await f.client + .get('/api/agents/v1/responses/resp_unavailable') + .auth(f.credential.key, { type: 'bearer' }) + .expect(503); + expect(result.body.error.code).toBe('response_service_unavailable'); + expect(JSON.stringify(result.body)).not.toContain('private journal query'); + expect(read).toHaveBeenCalledTimes(1); + expect(f.ordinaryRetrievals).toEqual([]); + } finally { + read.mockRestore(); + } +}); + +it('cancels an unstarted response durably and prevents a delayed worker from starting the provider', async () => { + const f = await fixture(); + const record = await reserveUnstarted(f.credential.id); + await expect(scoped(() => f.host.retireResponse(record, record.responseId))).resolves.toBe( + 'stopped', + ); + const retired = await f.client + .get(`/api/agents/v1/responses/${record.responseId}`) + .auth(f.credential.key, { type: 'bearer' }); + expect(retired.body.status).toBe('cancelled'); + const delayed = await f.client + .post('/api/agents/v1/responses') + .set('x-lc-response-worker', '1') + .auth(mintBackgroundResponseWorkerToken(record, 'isolated-background-host-secret'), { + type: 'bearer', + }) + .send({}); + expect(delayed.status).toBe(409); + expect(f.inputs).toHaveLength(0); + expect((await scoped(() => storage.read(record, record.responseId)))?.status).toBe('cancelled'); +}); + +it('does not turn a missing admitted job into a fabricated stop confirmation', async () => { + const f = await fixture(); + const record = await reserveUnstarted(f.credential.id); + await scoped(() => + storage.publish(record, record.responseId, record.revision, { + status: 'in_progress', + responseJSON: JSON.stringify({}), + generationCreatedAt: 1234, + }), + ); + await expect(scoped(() => f.host.retireResponse(record, record.responseId))).resolves.toBe( + 'pending', + ); + expect((await scoped(() => storage.read(record, record.responseId)))?.status).toBe('in_progress'); +}); + +it('denies an idempotent POST replay through a replacement key after the original key is revoked', async () => { + const f = await fixture(); + const body = { model: 'agent', input: 'Original credential output', background: true }; + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .set('Idempotency-Key', 'original-source') + .send(body) + .expect(202); + await f.readUntil(created.body.id, 'completed'); + const replacement = await scoped(() => + db.createAgentApiKey({ userId: user._id, name: 'replacement' }), + ); + await scoped(() => db.deleteAgentApiKey(f.credential.id, String(user._id))); + const replay = await f.client + .post('/api/agents/v1/responses') + .auth(replacement.key, { type: 'bearer' }) + .set('Idempotency-Key', 'original-source') + .send(body) + .expect(403); + expect(JSON.stringify(replay.body)).not.toContain('# Real Responses lifecycle'); + expect(f.inputs).toHaveLength(1); + expect(f.errors).toEqual([]); +}); + +it('reports a deterministic envelope depth error as a safe invalid request', async () => { + const f = await fixture(); + let nested: Record = {}; + for (let depth = 0; depth < 70; depth++) nested = { nested }; + const result = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ + model: 'agent', + input: 'Sensitive submitted input', + background: true, + metadata: nested, + }) + .expect(400); + expect(JSON.stringify(result.body)).not.toContain('Sensitive submitted input'); + expect(f.inputs).toEqual([]); + expect(f.errors).toEqual([]); +}); + +it('preserves the native recovery marker when a retained paused execution is unavailable', async () => { + const f = await fixture(); + const { owner, id } = await ownerPause(f.credential.id); + const job = await GenerationJobManager.getJob(id); + expect(job).toBeDefined(); + expect(await jobStore.deleteJob(id, job!.createdAt)).toBe(true); + const view = await scoped(() => f.host.readInteractiveReview(owner, id)); + expect(view).toMatchObject({ + responseId: id, + recovery: 'execution_state_unavailable', + oauth: [], + }); +}); + +const concreteInterface = Object.values(networkInterfaces()) + .flat() + .find((entry) => entry?.family === 'IPv4' && !entry.internal)?.address; +if (concreteInterface) { + it('executes through the capable listener bound only to a concrete interface', async () => { + const f = await fixture(undefined, undefined, concreteInterface); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Concrete listener', background: true }) + .expect(202); + expect((await f.readUntil(created.body.id, 'completed')).body.output[0].content[0].text).toBe( + '# Real Responses lifecycle', + ); + expect(f.errors).toEqual([]); + }); +} else { + it.skip('requires a concrete network interface for listener integration', () => { + expect(concreteInterface).toBeDefined(); + }); +} + +it.each([undefined, null, ''])( + 'executes a trusted legacy tenantless owner with tenant %s', + async (legacyTenant) => { + tenantId = undefined; + user = await runAsSystem(() => + mongoose.models.User.create({ + email: 'tenantless-response-host@example.test', + role: 'USER', + provider: 'local', + tenantId: legacyTenant, + }), + ); + const f = await fixture(); + const created = await f.client + .post('/api/agents/v1/responses') + .auth(f.credential.key, { type: 'bearer' }) + .send({ model: 'agent', input: 'Legacy tenantless owner', background: true }) + .expect(202); + expect((await f.readUntil(created.body.id, 'completed')).body.output[0].content[0].text).toBe( + '# Real Responses lifecycle', + ); + expect(f.errors).toEqual([]); + }, +); diff --git a/packages/api/src/agents/responses/host.ts b/packages/api/src/agents/responses/host.ts new file mode 100644 index 0000000000..23700ca5fb --- /dev/null +++ b/packages/api/src/agents/responses/host.ts @@ -0,0 +1,330 @@ +import { + AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1, + createChatExpirationDate, + createFallbackRetentionDate, +} from '@librechat/data-schemas'; +import type { + IUser, + ResponseOwner, + ResponseStorage, + StoredResponseRecord, + ConversationMethods, +} from '@librechat/data-schemas'; +import type { ResponseReviewView } from 'librechat-data-provider'; +import type { Request, RequestHandler } from 'express'; +import type { ResponseDecisionDependencies, ResponseDecisionSubmission } from './decision'; +import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import type { BackgroundResponseAuthorityDependencies } from './authority'; +import type { BackgroundResponseTransportDependencies } from './transport'; +import type { BackgroundResponseWorkerDependencies } from './worker'; +import type { AgentTriggerService } from '../triggers/service'; +import type { BackgroundResponseResult } from './background'; +import type { BackgroundResponseQueue } from './queue'; +import type { ServerRequest } from '~/types/http'; +import type { BoundAddress } from '~/app/origin'; +import { + createResponseCompletion, + createResponseExecutionRelease, + createResponseExecutionConfirmation, +} from './completion'; +import { createBackgroundResponseWorkerAuth, getBackgroundResponseWorkerRecord } from './worker'; +import { createBackgroundResponseTransport, createResponseResumeTransport } from './transport'; +import { resolveImportRetentionFields, isTemporaryRecord } from '~/conversations/retention'; +import { createBackgroundResponseQueue, createResponseResumeQueue } from './queue'; +import { createBackgroundResponseAuthority } from './authority'; +import { createBackgroundResponseService } from './background'; +import { createBackgroundResponseHttpHandlers } from './http'; +import { createResponseDecisionSubmission } from './decision'; +import { resolveResponseContinuation } from './persistence'; +import { cleanupResponseCheckpoint } from './checkpoint'; +import { createResponseRetirement } from './retirement'; +import { createResponseReview } from './review'; +import { responseTenantId } from './scope'; + +export interface BackgroundResponsesHostDependencies + extends BackgroundResponseAuthorityDependencies { + storage: ResponseStorage; + enqueue: AgentTriggerService['enqueue']; + jobs: Pick< + GenerationJobManagerClass, + 'getJob' | 'abortJob' | 'getResumeState' | 'finishProviderTerminalPersistence' + >; + publicBaseURL?: string; + getMessage: Parameters[2]['getMessage']; + getConvoRetention: ConversationMethods['getConvoRetention']; + signingSecret: () => string; + onError: (error: unknown) => void; + fetch?: BackgroundResponseTransportDependencies['fetch']; + decisions?: Pick; + /** Install together: owner validation, durable dispatch, and same-checkpoint worker admission. */ + interactive?: { + validate: ResponseDecisionDependencies['validate']; + admitResume: NonNullable; + }; +} + +export interface BackgroundResponsesHost { + /** Trusted transports must derive this owner from current actor/source authority. Never expose directly as service-token authority. */ + readInteractiveReview: ( + owner: ResponseOwner, + responseId: string, + ) => Promise; + submitInteractiveReview: ResponseDecisionSubmission; + review: RequestHandler; + submitReview: RequestHandler; + cancelReview: RequestHandler; + retireResponse: (owner: ResponseOwner, responseId: string) => Promise<'stopped' | 'pending'>; + readResponse: (owner: ResponseOwner, responseId: string) => Promise; + cancelResponse: (owner: ResponseOwner, responseId: string) => Promise; + sourceDispatchers: BackgroundResponseQueue['sourceDispatchers']; + initialize: (address: BoundAddress | string | null) => void; + authenticate: (ordinary: RequestHandler) => RequestHandler; + handlers: (ordinary: { create: RequestHandler; retrieve: RequestHandler }) => { + create: RequestHandler; + retrieve: RequestHandler; + cancel: RequestHandler; + }; +} + +/** A scoped execution credential is not the owner's interactive browser authority. */ +function ownerReviewUser(req: Request): IUser | undefined { + if ( + getBackgroundResponseWorkerRecord(req) || + ('_isAgentTrigger' in req && req._isAgentTrigger === true) || + ('apiKeyId' in req && req.apiKeyId != null) + ) + return; + return (req as ServerRequest).user; +} + +export function createBackgroundResponsesHost( + deps: BackgroundResponsesHostDependencies, +): BackgroundResponsesHost { + if (deps.decisions && deps.interactive) + throw new Error('Response decision transport must have one owner'); + let listener: BoundAddress | undefined; + const authorize = createBackgroundResponseAuthority(deps); + const review = createResponseReview(deps); + const completion = createResponseCompletion({ + storage: deps.storage, + cleanupCheckpoint: cleanupResponseCheckpoint, + releaseExecution: createResponseExecutionRelease(deps.jobs), + confirmExecution: createResponseExecutionConfirmation(deps.jobs), + }); + const retire = createResponseRetirement({ + storage: deps.storage, + jobs: deps.jobs, + now: deps.now, + cleanupCheckpoint: cleanupResponseCheckpoint, + recoverTerminal: completion.finish, + }); + const transportDependencies: BackgroundResponseTransportDependencies = { + storage: deps.storage, + signingSecret: deps.signingSecret, + fetch: deps.fetch, + getOrigin: () => { + if (!listener) throw new Error('Background Responses local listener is not initialized'); + return listener; + }, + acknowledgementTimeoutMs: async (record) => { + const user = await deps.findUser(record.userId); + if (!user || responseTenantId(user.tenantId) !== responseTenantId(record.tenantId)) + throw new Error('Response principal unavailable'); + const timeout = (await deps.getConfig(user)).endpoints?.agents?.remoteApi?.background + ?.acknowledgementTimeoutMs; + if (timeout == null) + throw new Error('Background response acknowledgement policy unavailable'); + return timeout; + }, + }; + const transport = createBackgroundResponseTransport(transportDependencies); + const queue = createBackgroundResponseQueue({ + storage: deps.storage, + enqueue: deps.enqueue, + authorize: (record) => authorize(record, record.agentId, record.sourceKeyId), + execute: transport, + workerCapability: deps.interactive + ? AGENT_TRIGGER_WORKER_CAPABILITY_RESPONSES_RESUME_V1 + : undefined, + }); + const resumeQueue = deps.interactive + ? createResponseResumeQueue({ + storage: deps.storage, + enqueue: deps.enqueue, + authorize: (record) => authorize(record, record.agentId, record.sourceKeyId), + resume: createResponseResumeTransport({ ...transportDependencies, jobs: deps.jobs }), + }) + : undefined; + const decisions = + deps.interactive && resumeQueue + ? { validate: deps.interactive.validate, enqueue: resumeQueue.enqueue } + : deps.decisions; + + const submitInteractiveReview: ResponseDecisionSubmission = async (owner, responseId, input) => { + if (!decisions) return { ok: false, status: 409, code: 'response_decision_unavailable' }; + return createResponseDecisionSubmission({ + storage: deps.storage, + jobs: deps.jobs, + now: deps.now, + ...decisions, + authorize: (record) => authorize(record, record.agentId, record.sourceKeyId), + })(owner, responseId, input); + }; + + function service(req?: Request) { + const user = (req as ServerRequest | undefined)?.user; + return createBackgroundResponseService({ + storage: deps.storage, + enqueue: queue.enqueue, + deliveryKey: queue.deliveryKey, + authorize: (owner, agentId, keyId) => authorize(owner, agentId, keyId, user), + resolveContinuation: async (owner, previous) => { + const continuation = await resolveResponseContinuation(previous, owner.userId, { + getMessage: deps.getMessage, + }); + const stored = previous + ? await deps.getConvoRetention(owner.userId, continuation.conversationId) + : null; + if (previous && !stored) throw new Error('Response continuation is unavailable'); + const principal = user ?? (await deps.findUser(owner.userId)); + if (!principal) throw new Error('Response principal unavailable'); + const config = await deps.getConfig(principal); + const policy = resolveImportRetentionFields( + config.interfaceConfig, + { + createChatExpirationDate, + createFallbackRetentionDate, + }, + { sourceIsTemporary: isTemporaryRecord(stored ?? {}) }, + ); + let expiredAt = stored?.expiredAt ?? policy.expiredAt; + if (policy.expiredAt && (!expiredAt || policy.expiredAt < expiredAt)) + expiredAt = policy.expiredAt; + if (!expiredAt && stored && isTemporaryRecord(stored)) + expiredAt = createChatExpirationDate(config.interfaceConfig, true); + return { + ...continuation, + ...(expiredAt ? { expiredAt } : {}), + isTemporary: policy.isTemporary ?? isTemporaryRecord(stored ?? {}), + }; + }, + // Runtime publication is authoritative; observers never reconstruct a terminal reply from progress. + observe: review.observe, + recoverTerminal: completion.finish, + cancel: async (record) => { + await retire(record, record.responseId); + }, + }); + } + + return { + submitInteractiveReview, + readInteractiveReview: async (owner, responseId) => { + const record = await deps.storage.read(owner, responseId); + if (!record || !(await authorize(record, record.agentId, record.sourceKeyId))) return null; + const view = await review.privateView(await completion.finish(record)); + // OAuth URLs stay behind the existing owner-authenticated browser flow. + return { + responseId: view.responseId, + status: view.status, + ...(view.recovery ? { recovery: view.recovery } : {}), + oauth: [], + pendingAction: view.pendingAction, + pendingDecision: view.pendingDecision, + canSubmitDecision: decisions != null, + }; + }, + review: async (req, res) => { + res.set('Cache-Control', 'no-store'); + res.set('Referrer-Policy', 'no-referrer'); + const user = ownerReviewUser(req); + if (!user?.id) return void res.status(401).json({ error: 'unauthorized' }); + try { + const owner = { userId: user.id, tenantId: responseTenantId(user.tenantId) }; + const record = await deps.storage.read(owner, req.params.id); + if (!record || !(await authorize(owner, record.agentId, record.sourceKeyId, user))) + return void res.status(404).json({ error: 'response_not_found' }); + const recovered = await completion.finish(record); + res.json({ + ...(await review.privateView(recovered)), + canSubmitDecision: decisions != null, + }); + } catch (error) { + deps.onError(error); + res.status(503).json({ error: 'response_service_unavailable' }); + } + }, + submitReview: async (req, res) => { + res.set('Cache-Control', 'no-store'); + res.set('Referrer-Policy', 'no-referrer'); + const user = ownerReviewUser(req); + if (!user?.id) return void res.status(401).json({ error: 'unauthorized' }); + if (!decisions) return void res.status(503).json({ error: 'response_decision_unavailable' }); + const submit = createResponseDecisionSubmission({ + storage: deps.storage, + jobs: deps.jobs, + now: deps.now, + ...decisions, + authorize: (record) => authorize(record, record.agentId, record.sourceKeyId, user), + }); + try { + const result = await submit( + { userId: user.id, tenantId: responseTenantId(user.tenantId) }, + req.params.id, + req.body, + ); + if (!result.ok) return void res.status(result.status).json({ error: result.code }); + res.status(202).json(result.receipt); + } catch (error) { + deps.onError(error); + res.status(503).json({ error: 'response_service_unavailable' }); + } + }, + cancelReview: async (req, res) => { + res.set('Cache-Control', 'no-store'); + res.set('Referrer-Policy', 'no-referrer'); + const user = ownerReviewUser(req); + if (!user?.id) return void res.status(401).json({ error: 'unauthorized' }); + try { + const result = await service(req).cancel( + { userId: user.id, tenantId: responseTenantId(user.tenantId) }, + req.params.id, + ); + if (!result.ok) return void res.status(result.status).json({ error: result.code }); + res.json(result.response); + } catch (error) { + deps.onError(error); + res.status(503).json({ error: 'response_service_unavailable' }); + } + }, + retireResponse: retire, + readResponse: (owner, responseId) => service().retrieve(owner, responseId), + cancelResponse: (owner, responseId) => service().cancel(owner, responseId), + sourceDispatchers: new Map([ + ...queue.sourceDispatchers, + ...(resumeQueue?.sourceDispatchers ?? []), + ]), + initialize: (address: BoundAddress | string | null) => { + listener = address != null && typeof address !== 'string' ? { ...address } : undefined; + }, + authenticate: (ordinary: RequestHandler) => + createBackgroundResponseWorkerAuth( + { + storage: deps.storage, + signingSecret: deps.signingSecret, + findUser: deps.findUser, + onError: deps.onError, + admitResume: deps.interactive?.admitResume, + authorize: (record: StoredResponseRecord, user: IUser) => + authorize(record, record.agentId, record.sourceKeyId, user), + }, + ordinary, + ), + handlers: (ordinary: { create: RequestHandler; retrieve: RequestHandler }) => + createBackgroundResponseHttpHandlers({ + service, + ordinary, + onError: deps.onError, + }), + }; +} diff --git a/packages/api/src/agents/responses/http.ts b/packages/api/src/agents/responses/http.ts new file mode 100644 index 0000000000..f46f0380d4 --- /dev/null +++ b/packages/api/src/agents/responses/http.ts @@ -0,0 +1,127 @@ +import { randomUUID } from 'node:crypto'; +import type { Request, RequestHandler, Response as HttpResponse } from 'express'; +import type { ResponseOwner } from '@librechat/data-schemas'; +import type { createBackgroundResponseService, BackgroundResponseResult } from './background'; +import type { ServerRequest } from '~/types/http'; +import { validateResponseRequest, sendResponsesErrorResponse } from './service'; +import { createAgentRunEnvelope, AgentRunEnvelopeError } from '../envelope'; +import { getBackgroundResponseWorkerRecord } from './worker'; +import { responseTenantId } from './scope'; + +export interface BackgroundResponseHttpDependencies { + service: (req: Request) => ReturnType; + /** Foreground-only fallback; canonical background history remains journal-owned. */ + ordinary: { create: RequestHandler; retrieve: RequestHandler }; + onError: (error: unknown) => void; +} + +function owner(req: Request): ResponseOwner | null { + const user = (req as ServerRequest).user; + return user?.id ? { userId: user.id, tenantId: responseTenantId(user.tenantId) } : null; +} + +function send(res: HttpResponse, result: BackgroundResponseResult, accepted = false): void { + if (result.ok) return void res.status(accepted ? 202 : 200).json(result.response); + return void sendResponsesErrorResponse( + res, + result.status, + result.status === 404 ? 'Response not found.' : 'The response request could not be accepted.', + result.status === 404 ? 'not_found' : 'invalid_request', + result.code, + ); +} + +/** Install behind the existing remote auth, terms, config and feature/Agent permission checks. */ +export function createBackgroundResponseHttpHandlers(deps: BackgroundResponseHttpDependencies): { + create: RequestHandler; + retrieve: RequestHandler; + cancel: RequestHandler; +} { + const unavailable = (res: HttpResponse, error: unknown) => { + deps.onError(error); + return void sendResponsesErrorResponse( + res, + 503, + 'Response service is unavailable.', + 'server_error', + 'response_service_unavailable', + ); + }; + return { + create: async (req, res, next) => { + if (getBackgroundResponseWorkerRecord(req) || req.body?.background !== true) + return deps.ordinary.create(req, res, next); + const validation = validateResponseRequest(req.body); + if (!validation.valid || !validation.request) + return void sendResponsesErrorResponse( + res, + 400, + validation.error ?? 'Invalid response request.', + 'invalid_request', + ); + const principal = owner(req); + const key = (req as Request & { apiKeyId?: { toString(): string } }).apiKeyId; + if (!principal || !key) + return void sendResponsesErrorResponse( + res, + 400, + 'Background execution requires an Agent API key.', + 'invalid_request', + 'background_api_key_required', + ); + const idempotency = req.headers['idempotency-key']; + if (idempotency != null && typeof idempotency !== 'string') + return void sendResponsesErrorResponse( + res, + 400, + 'Invalid Idempotency-Key.', + 'invalid_request', + ); + try { + const envelope = createAgentRunEnvelope({ + protocol: 'responses', + requestId: randomUUID(), + receivedAt: Date.now(), + principal: { id: principal.userId, tenantId: principal.tenantId }, + payload: validation.request, + }); + return send( + res, + await deps.service(req).create(envelope, idempotency, key.toString()), + true, + ); + } catch (error) { + if (error instanceof AgentRunEnvelopeError) + return void sendResponsesErrorResponse( + res, + 400, + 'Invalid response request.', + 'invalid_request', + ); + return unavailable(res, error); + } + }, + retrieve: async (req, res, next) => { + const principal = owner(req); + if (!principal) + return void sendResponsesErrorResponse(res, 401, 'Unauthorized', 'invalid_request'); + try { + const result = await deps.service(req).lookup(principal, req.params.id); + if (result == null) return deps.ordinary.retrieve(req, res, next); + return send(res, result); + } catch (error) { + return unavailable(res, error); + } + }, + cancel: async (req, res) => { + const principal = owner(req); + if (!principal) + return void sendResponsesErrorResponse(res, 401, 'Unauthorized', 'invalid_request'); + try { + return send(res, await deps.service(req).cancel(principal, req.params.id)); + } catch (error) { + return unavailable(res, error); + } + }, + }; +} diff --git a/packages/api/src/agents/responses/index.ts b/packages/api/src/agents/responses/index.ts index 7e8f681292..a26a71380b 100644 --- a/packages/api/src/agents/responses/index.ts +++ b/packages/api/src/agents/responses/index.ts @@ -14,7 +14,7 @@ export { resolveResponseContinuation, } from './persistence'; export { createBackgroundResponseService } from './background'; - +export { createBackgroundResponsesHost } from './host'; export { createResponseExecutionRuntime } from './runtime'; export { createResponseDecisionSubmission } from './decision'; export { createResponseResumeValidation, createResponseResumeAdmission } from './resume'; diff --git a/packages/api/src/agents/responses/stored.ts b/packages/api/src/agents/responses/stored.ts index 014a46bf68..41433ac820 100644 --- a/packages/api/src/agents/responses/stored.ts +++ b/packages/api/src/agents/responses/stored.ts @@ -97,6 +97,7 @@ const receiptSchema = z.object({ createdAt: z.number(), completedAt: z.number().nullable(), }); +const backgroundReceiptSchema = z.object({ background: z.literal(true) }); const textPart = z.object({ type: z.literal(ContentTypes.TEXT), text: z.string() }); const toolPart = z.object({ type: z.literal(ContentTypes.TOOL_CALL), @@ -120,6 +121,10 @@ interface SavedConversation { agent_id?: string; model?: string; } + +function isBackgroundMessage(message: SavedMessage): boolean { + return backgroundReceiptSchema.safeParse(message.metadata?.responseReceipt).success; +} export interface StoredForegroundResponseDependencies { getMessage: (query: { user: string; messageId: string }) => Promise; getConvo: (userId: string, conversationId: string) => Promise; @@ -216,7 +221,7 @@ function restoreOutputIdentity( return output; } -/** Both response-ID and legacy conversation-ID reads remain owner/tenant scoped. */ +/** Foreground-only response-ID and legacy conversation-ID reads remain owner/tenant scoped. */ export function createStoredForegroundResponseReader( deps: StoredForegroundResponseDependencies, ): RequestHandler { @@ -239,7 +244,10 @@ export function createStoredForegroundResponseReader( : undefined; if ( selected && - (selected.isCreatedByUser || selected.isUserSubmitted || !selected.conversationId) + (selected.isCreatedByUser || + selected.isUserSubmitted || + !selected.conversationId || + isBackgroundMessage(selected)) ) return missing(); const conversationId = selected?.conversationId ?? responseId; @@ -248,7 +256,7 @@ export function createStoredForegroundResponseReader( selected ? Promise.resolve([selected]) : deps.getMessages({ user: userId, conversationId }), ]); const message = selectStoredResponse(messages ?? [], responseId, selected != null)[0]; - if (!conversation || !message) return missing(); + if (!conversation || !message || isBackgroundMessage(message)) return missing(); const receipt = receiptSchema.safeParse(message.metadata?.responseReceipt); const context = createResponseContext( { @@ -281,7 +289,6 @@ export function createStoredForegroundResponseReader( : createResponseAccountingCollector().snapshot(); if (receipt.success) { response.status = receipt.data.status ?? response.status; - response.background = receipt.data.background ?? response.background; response.error = receipt.data.error ?? null; } response.previous_response_id = receipt.success ? receipt.data.previousResponseId : null; diff --git a/packages/api/src/agents/responses/transport.spec.ts b/packages/api/src/agents/responses/transport.spec.ts index 06288033ff..a9f4e14e85 100644 --- a/packages/api/src/agents/responses/transport.spec.ts +++ b/packages/api/src/agents/responses/transport.spec.ts @@ -8,6 +8,7 @@ import { } from '@librechat/data-schemas'; import type { ResponseStorage, StoredResponseRecord } from '@librechat/data-schemas'; import type { BackgroundResponseTransportDependencies } from './transport'; +import type { BoundAddress } from '~/app/origin'; import { createBackgroundResponseTransport } from './transport'; let mongo: MongoMemoryReplSet; @@ -65,7 +66,7 @@ async function enroll() { ); if (!result.ok) throw new Error('Enrollment failed'); } -function transport(fetch: Fetch, origin = 'http://127.0.0.1:3088') { +function transport(fetch: Fetch, origin: string | BoundAddress = 'http://127.0.0.1:3088') { return createBackgroundResponseTransport({ storage, fetch, @@ -170,3 +171,25 @@ it('refuses remote or credential-bearing worker origins before making a request' } expect(calls).toBe(0); }); + +it('uses an initialized concrete listener capability without accepting the equivalent arbitrary URL', async () => { + const fetch: Fetch = async (url) => { + expect(String(url)).toBe('http://192.0.2.7:3088/api/agents/v1/responses'); + await enroll(); + return Response.json(ack, { status: 202 }); + }; + await expect(scoped(() => transport(fetch, 'http://192.0.2.7:3088')(record))).rejects.toThrow( + 'capable local listener', + ); + await expect( + scoped(() => transport(fetch, { address: '192.0.2.7', family: 'IPv4', port: 3088 })(record)), + ).resolves.toEqual({ status: 'started', generationCreatedAt: 1234 }); +}); + +it('rejects a hostname masquerading as a bound listener before dispatch', async () => { + const fetch = jest.fn, Parameters>(); + await expect( + scoped(() => transport(fetch, { address: 'example.test', port: 3088 })(record)), + ).rejects.toThrow('capable local listener'); + expect(fetch).not.toHaveBeenCalled(); +}); diff --git a/packages/api/src/agents/responses/transport.ts b/packages/api/src/agents/responses/transport.ts index 93f2a3f562..737a33e776 100644 --- a/packages/api/src/agents/responses/transport.ts +++ b/packages/api/src/agents/responses/transport.ts @@ -1,18 +1,22 @@ import { z } from 'zod'; +import { isIP } from 'node:net'; import type { StoredResponseRecord, ResponseStorage } from '@librechat/data-schemas'; import type { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; import type { BackgroundResponseQueueDependencies } from './queue'; +import type { BoundAddress } from '~/app/origin'; import { mintBackgroundResponseWorkerToken, mintResponseResumeWorkerToken, RESPONSE_WORKER_TOKEN_EXPIRED, } from './worker'; import { AgentTriggerExecutionError } from '../triggers/host'; +import { selfOriginFromAddress } from '~/app/origin'; import { responseTenantId } from './scope'; export interface BackgroundResponseTransportDependencies { storage: ResponseStorage; - getOrigin: () => string; + /** A BoundAddress must come from the capable process's initialized server.address(). */ + getOrigin: () => string | BoundAddress; signingSecret: () => string; acknowledgementTimeoutMs: (record: StoredResponseRecord) => Promise; fetch?: ( @@ -96,12 +100,17 @@ function createResponseWorkerTransport( status: 'started' | 'resumed' | 'settled'; generationCreatedAt?: number; }> => { - const origin = new URL(deps.getOrigin()); + const listener = deps.getOrigin(); + const initialized = typeof listener !== 'string'; + const localOrigin = initialized ? selfOriginFromAddress(listener) : listener; + if (!localOrigin || (initialized && !isIP(listener.address))) + throw new Error('Background Responses requires the capable local listener'); + const origin = new URL(localOrigin); if ( !['http:', 'https:'].includes(origin.protocol) || origin.username || origin.password || - !['127.0.0.1', '::1', '[::1]', 'localhost'].includes(origin.hostname) + (!initialized && !['127.0.0.1', '::1', '[::1]', 'localhost'].includes(origin.hostname)) ) throw new Error('Background Responses requires the capable local listener'); const timeoutMs = await deps.acknowledgementTimeoutMs(record); diff --git a/packages/api/src/agents/responses/worker.runtime.spec.ts b/packages/api/src/agents/responses/worker.runtime.spec.ts index 4af73d4a94..57b69340c2 100644 --- a/packages/api/src/agents/responses/worker.runtime.spec.ts +++ b/packages/api/src/agents/responses/worker.runtime.spec.ts @@ -2,8 +2,8 @@ import express from 'express'; import jwt from 'jsonwebtoken'; import request from 'supertest'; import mongoose from 'mongoose'; -import { FileSources } from 'librechat-data-provider'; import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { FileSources, PermissionBits } from 'librechat-data-provider'; import { createModels, createMethods, @@ -43,7 +43,10 @@ import { GenerationJobManager } from '~/stream'; import { createResponseTurn } from './turn'; // Load the compiled entry as a suite dependency, outside the settlement test's deadline. -jest.requireActual('@librechat/api'); +const compiled = jest.requireActual<{ + createStoredForegroundResponseReader: typeof import('./stored').createStoredForegroundResponseReader; + createBackgroundResponsesHost: typeof import('./host').createBackgroundResponsesHost; +}>('@librechat/api'); let mongo: MongoMemoryReplSet; let storage: ResponseStorage; @@ -1348,17 +1351,49 @@ describe('background Responses execution handoff', () => { const readApp = express(); const methods = createMethods(mongoose); readApp.use((req, _res, next) => { - Object.assign(req, { user: { id: owner.userId } }); + Object.assign(req, { user }); tenantStorage.run(owner, next); }); - // Exercise the built export wired by the CJS controller, against the actual committed rows. - const compiled = jest.requireActual<{ - createStoredForegroundResponseReader: typeof import('./stored').createStoredForegroundResponseReader; - }>('@librechat/api'); - readApp.get( - '/:id', - compiled.createStoredForegroundResponseReader({ ...methods, logger: { error: jest.fn() } }), - ); + // Exercise the compiled host and its actual foreground fallback over committed history. + const readHost = compiled.createBackgroundResponsesHost({ + storage, + jobs: GenerationJobManager, + findUser: async () => user, + getKey: async () => ({}), + getRole: async () => + new mongoose.models.Role({ name: 'USER', permissions: { REMOTE_AGENTS: { USE: true } } }), + getConfig: async () => ({ + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + endpoints: { + agents: { + remoteApi: { background: { enabled: true, acknowledgementTimeoutMs: 1000 } }, + }, + }, + }), + isPrincipalActive: async () => true, + getAgentPermissions: async () => PermissionBits.VIEW, + getMessage: methods.getMessage, + getConvoRetention: methods.getConvoRetention, + enqueue: async () => { + throw new Error('Retrieval must not enqueue another provider'); + }, + signingSecret: () => secret, + onError: (error) => { + errors.push(error); + }, + }); + const readHandlers = readHost.handlers({ + create: (_req, res) => { + res.sendStatus(500); + }, + retrieve: compiled.createStoredForegroundResponseReader({ + ...methods, + logger: { error: jest.fn() }, + }), + }); + readApp.get('/:id', readHandlers.retrieve); const read = await request(readApp) .get('/' + record.responseId) .expect(200); @@ -1367,6 +1402,17 @@ describe('background Responses execution handoff', () => { expect(read.body.error).toEqual(JSON.parse(saved.responseJSON!).error); expect(JSON.stringify(read.body)).not.toContain('private provider diagnostics'); + await scope(() => mongoose.models.ResponseScope.updateMany({}, { $set: { closed: true } })); + await request(readApp) + .get('/' + record.responseId) + .expect(404); + await scope(() => + mongoose.models.StoredResponse.deleteOne({ responseId: record.responseId }), + ); + await request(readApp) + .get('/' + record.responseId) + .expect(404); + expect( await scope(() => mongoose.models.Message.findOne({