diff --git a/packages/api/src/channels/authority.spec.ts b/packages/api/src/channels/authority.spec.ts new file mode 100644 index 0000000000..69f4ce1886 --- /dev/null +++ b/packages/api/src/channels/authority.spec.ts @@ -0,0 +1,1550 @@ +import express from 'express'; +import request from 'supertest'; +import mongoose from 'mongoose'; +import { Readable } from 'node:stream'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { createHash, createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; +import { + PermissionBits, + Permissions, + PermissionTypes, + FileSources, + channelConfigSchema, +} from 'librechat-data-provider'; +import { + createModels, + createMethods, + createChannelIdentityStorage, + createChannelRunStorage, + createResponseStorage, + hashToken, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { + ChannelRunAdmission, + ChannelIdentityResult, + ChannelRunSnapshot, + ChannelRunSummary, +} from 'librechat-data-provider'; +import type { IUser, ChannelRunRecord } from '@librechat/data-schemas'; +import type { ChannelResponseCredentialDependencies } from './credentials'; +import type { BackgroundResponsesHost } from '~/agents/responses/host'; +import { createChannelAuthority, channelExternalIdentity } from './authority'; +import { createChannelResponseCredentials } from './credentials'; +import { createChannelArtifactService } from './artifacts'; +import { ChannelExecutionError } from './execution'; +import { createChannelHandlers } from './service'; +import { createChannelRunHandlers } from './runs'; + +let mongo: MongoMemoryReplSet; +let db: ReturnType; +let models: ReturnType; +let authority: ReturnType; +let storage: ReturnType; +let users: IUser[]; +let acl: Map; +let invoked: ChannelRunRecord[]; +let advanceError: ChannelExecutionError | undefined; +let retired: ChannelRunRecord[]; +let retirement: 'stopped' | 'pending'; +let artifactRead: () => Promise; +let nextSnapshot: ChannelRunSnapshot; +let summarize: jest.Mock, [ChannelRunRecord, ChannelRunSnapshot]>; +let responseCredential: jest.Mock, [ChannelRunRecord]>; +let app: express.Express; +let fakeNow: number | undefined; +let interactive: Pick; +const tenantId = 'tenant-channel-authority'; +const token = 'test-channel-service-bearer-32-characters'; +const scoped = (work: () => Promise) => () => tenantStorage.run({ tenantId }, work); +const value = (result: ChannelIdentityResult): T => { + if (!result.ok) throw new Error(result.error.code); + return result.value; +}; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await mongoose.connect(mongo.getUri()); + models = createModels(mongoose); + db = createMethods(mongoose); + storage = createChannelRunStorage(mongoose); + await Promise.all([ + models.ChannelIdentity.init(), + models.ChannelRun.init(), + models.ChannelLane.init(), + ]); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await Promise.all([ + models.ChannelIdentity.deleteMany({}), + models.StoredResponse.deleteMany({}), + models.ResponseScope.deleteMany({}), + models.ChannelRun.deleteMany({}), + models.ChannelLane.deleteMany({}), + models.AgentChannel.deleteMany({}), + models.User.deleteMany({}), + models.Role.deleteMany({}), + models.File.deleteMany({}), + models.Message.deleteMany({}), + ]); + }); + await tenantStorage.run({ tenantId }, async () => { + users = await models.User.create([ + { email: 'a@example.test', name: 'Account A', role: 'USER' }, + { email: 'b@example.test', name: 'Account B', role: 'USER' }, + ]); + await models.Role.create({ + name: 'USER', + permissions: { + [PermissionTypes.REMOTE_AGENTS]: { [Permissions.USE]: true }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: true }, + }, + }); + await db.createAgentChannel({ + channelId: 'channel', + tenantId, + agentId: 'agent', + createdBy: users[0].id, + type: 'slack', + status: 'active', + credentialMode: 'per_user', + revision: 'revision-1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' }, + }); + }); + acl = new Map(users.map((user) => [user.id, PermissionBits.VIEW])); + invoked = []; + fakeNow = undefined; + advanceError = undefined; + retired = []; + retirement = 'stopped'; + artifactRead = async () => {}; + nextSnapshot = { status: 'running', text: '', elapsedMs: 1 }; + summarize = jest.fn, [ChannelRunRecord, ChannelRunSnapshot]>( + async () => ({ + version: 1, + scope: 'actor_lane', + throughSequence: 1, + includesCurrent: true, + priorRuns: 0, + expectedRuns: 1, + observedRuns: 1, + costUSD: { coverage: 'unavailable', coveredRuns: 0 }, + elapsedMs: { coverage: 'complete', coveredRuns: 1, value: 12 }, + tokens: { coverage: 'unavailable', coveredRuns: 0 }, + }), + ); + authority = createChannelAuthority({ + channels: db, + identityStorage: createChannelIdentityStorage(mongoose), + getConfig: async () => ({ enabled: true }), + findUser: (id) => db.findUser({ _id: id }), + getRole: (name) => models.Role.findOne({ name }), + isPrincipalActive: async () => true, + getAgentPermissions: async (_id, user) => acl.get(user.id) ?? 0, + publicBaseUrl: 'http://127.0.0.1:3088', + }); + const handlers = createChannelHandlers({ + db, + serviceToken: () => token, + config: async () => ({ enabled: true }), + getAgent: async (id) => ({ id }), + deleteApiKey: async () => true, + verifySlackToken: async () => null, + logError: jest.fn(), + }); + responseCredential = jest.fn, [ChannelRunRecord]>(async () => 'sk-test'); + interactive = { + readInteractiveReview: jest.fn(async (_owner, responseId) => ({ + responseId, + status: 'in_progress' as const, + oauth: [], + canSubmitDecision: true, + pendingAction: { + actionId: 'native-action', + expiresAt: Date.now() + 60000, + payload: { type: 'ask_user_question' as const, question: { question: 'Private question' } }, + }, + })), + submitInteractiveReview: jest.fn(async (_owner, responseId, decision) => ({ + ok: true as const, + receipt: { + responseId, + actionId: (decision as { actionId: string }).actionId, + intentId: 'intent', + status: 'queued' as const, + }, + })), + }; + const runs = createChannelRunHandlers({ + now: () => fakeNow ?? Date.now(), + interactive, + responseCredential, + storage, + artifacts: createChannelArtifactService({ + readArtifacts: db.readResponseArtifacts, + readFile: async () => { + await artifactRead(); + return Readable.from(Buffer.from('file')); + }, + limits: async () => + channelConfigSchema.parse({ + artifactMaxBytes: 1000, + artifactMaxFiles: 1, + artifactReadTimeoutMs: 1000, + }), + }), + guard: handlers.service, + resolvePrincipal: authority.resolvePrincipal, + summarize: (run, snapshot) => summarize(run, snapshot), + publicBaseUrl: 'http://127.0.0.1:3088/base', + advance: async (run) => { + if (advanceError) throw advanceError; + invoked.push(run); + if (nextSnapshot.status === 'completed' && !run.execution?.responseId) { + const responseId = nextSnapshot.responseId ?? `resp_${run.runId}`; + const execution = run.execution!; + await models.StoredResponse.create({ + tenantId, + userId: run.principal.userId, + responseId, + requestKey: `channel:${run.runId}`, + sourceKeyId: execution.apiKeyId, + agentId: run.agentId, + fingerprint: 'fixture', + requestJSON: JSON.stringify({ + model: run.agentId, + input: run.admission.input, + background: true, + store: true, + }), + conversationId: execution.conversationId, + parentMessageId: execution.parentMessageId, + status: 'queued', + revision: 0, + }); + const bound = await storage.bindResponse( + tenantId, + { runId: run.runId, owner: run.leaseOwner!, epoch: run.leaseEpoch }, + responseId, + Date.now(), + ); + run.execution = bound.execution; + nextSnapshot.responseId = responseId; + } + return { snapshot: { ...nextSnapshot } }; + }, + retireForAuthorityLoss: async (run) => { + retired.push(run); + return retirement; + }, + }); + app = express(); + app.use(express.json()); + app.post('/service/:channelId/runs', runs.admit); + app.post('/service/:channelId/runs/claim', runs.claim); + app.post('/service/:channelId/runs/stop-thread', runs.stopThread); + app.post('/service/:channelId/runs/:runId/yield', runs.yieldLease); + app.post('/service/:channelId/runs/:runId/advance', runs.advance); + app.post('/service/:channelId/runs/:runId/response/prepare', runs.prepareResponse); + app.post('/service/:channelId/runs/:runId/response/bind', runs.bindResponse); + app.post('/service/:channelId/runs/:runId/cancel', runs.cancel); + app.post('/service/:channelId/runs/:runId/review', runs.review); + app.post('/service/:channelId/runs/:runId/review/decision', runs.submitReview); + app.post('/service/:channelId/runs/:runId/plan', runs.plan); + app.post('/service/:channelId/runs/:runId/settle', runs.settle); + app.post('/service/:channelId/runs/:runId/commit', runs.commit); + app.post('/service/:channelId/runs/:runId/progress/plan', runs.planProgress); + app.post('/service/:channelId/runs/:runId/progress/begin', runs.beginProgress); + app.post('/service/:channelId/runs/:runId/progress/settle', runs.settleProgress); + app.post('/service/:channelId/runs/:runId/delivery/reconcile', runs.reconcileDelivery); + app.post('/service/:channelId/runs/:runId/progress/reconcile', runs.reconcileProgress); + + app.post('/service/:channelId/runs/:runId/begin', runs.begin); + app.post('/service/:channelId/runs/:runId/artifacts/:artifactId', runs.artifact); +}); + +function event(actor = 'U1', eventId = `Ev-${actor}`): ChannelRunAdmission { + return { + eventId, + externalChannelId: 'C1', + externalThreadKey: 'C1:1.000001', + externalMessageKey: `C1:${eventId}`, + actor: { teamId: 'T1', appId: 'A1', externalUserId: actor }, + input: 'Use my own permissions', + }; +} + +async function bindFixtureResponse(run: ChannelRunRecord, responseId: string) { + const execution = run.execution!; + await models.StoredResponse.create({ + tenantId, + userId: run.principal.userId, + responseId, + requestKey: `channel:${run.runId}`, + sourceKeyId: execution.apiKeyId, + agentId: run.agentId, + fingerprint: 'fixture', + requestJSON: JSON.stringify({ + model: run.agentId, + input: run.admission.input, + background: true, + store: true, + }), + conversationId: execution.conversationId, + parentMessageId: execution.parentMessageId, + status: 'queued', + revision: 0, + }); + await models.Message.create({ + tenantId, + user: run.principal.userId, + conversationId: execution.conversationId, + messageId: responseId, + isCreatedByUser: false, + sender: 'Agent', + text: 'answer', + attachments: [], + }); + return storage.bindResponse( + tenantId, + { runId: run.runId, owner: run.leaseOwner!, epoch: run.leaseEpoch }, + responseId, + Date.now(), + ); +} + +function responseCredentials(overrides: Partial = {}) { + const key = randomBytes(32); + return createChannelResponseCredentials({ + storage, + responses: createResponseStorage(mongoose), + getChannel: db.getAgentChannelMetadata, + resolvePrincipal: authority.resolvePrincipal, + createKey: db.createAgentApiKey, + getKey: db.getAgentApiKeyById, + deleteKey: db.deleteAgentApiKey, + lifetimeMs: async () => 60000, + encrypt: async (value) => { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key, iv); + const encrypted = Buffer.concat([cipher.update(value, 'utf8'), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), encrypted]).toString('base64'); + }, + decrypt: async (value) => { + const encoded = Buffer.from(value, 'base64'); + const decipher = createDecipheriv('aes-256-gcm', key, encoded.subarray(0, 12)); + decipher.setAuthTag(encoded.subarray(12, 28)); + return Buffer.concat([decipher.update(encoded.subarray(28)), decipher.final()]).toString( + 'utf8', + ); + }, + ...overrides, + }); +} + +it( + 'provisions one encrypted, run-bound credential under concurrent gateway retries', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + const credentials = responseCredentials(); + const keys = await Promise.all([ + credentials.provision(run), + credentials.provision(run), + credentials.provision(run), + ]); + expect(new Set(keys).size).toBe(1); + expect(keys[0]).toMatch(/^sk-bound-/); + const saved = (await storage.read(tenantId, 'channel', admitted.body.runId))!; + expect(saved.execution?.encryptedApiKey).not.toContain(keys[0]); + expect(saved.execution?.protocol).toBe('responses'); + expect(await db.listAgentApiKeys(users[0].id)).toHaveLength(1); + expect(await db.validateAgentApiKey(keys[0])).toBeNull(); + const validated = await db.validateBoundAgentApiKey(keys[0]); + expect(validated?.binding).toEqual({ kind: 'channel_run', id: `channel/${run.runId}` }); + // The pre-binding validator used the raw-token hash. An old replica must not admit this key. + expect( + await mongoose.models.AgentApiKey.findOne({ keyHash: await hashToken(keys[0]) }), + ).toBeNull(); + await db.deleteAgentApiKey(saved.execution!.apiKeyId!, users[0].id); + await expect(credentials.provision(saved)).rejects.toThrow('authority changed'); + }), +); + +it( + 'prepares and binds a canonical Response without executing through the Channels service', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const route = `runs/${admitted.body.runId}/response`; + const prepared = await post(`${route}/prepare`).send({ fence: claimed.body.fence }).expect(200); + expect(prepared.body).toEqual({ + apiKey: 'sk-test', + requestKey: `channel:${admitted.body.runId}`, + request: { model: 'agent', input: event().input, background: true, store: true }, + }); + expect(invoked).toHaveLength(0); + await post(`${route}/bind`) + .send({ fence: claimed.body.fence, responseId: 'resp_missing' }) + .expect(409); + const reserved = await createResponseStorage(mongoose).reserve({ + tenantId, + userId: users[0].id, + responseId: `resp_${admitted.body.runId}`, + requestKey: prepared.body.requestKey, + fingerprint: 'request', + agentId: 'agent', + requestJSON: JSON.stringify(prepared.body.request), + conversationId: 'canonical-conversation', + parentMessageId: '00000000-0000-0000-0000-000000000000', + }); + if (!reserved.ok) throw new Error('Reservation failed'); + const body = { fence: claimed.body.fence, responseId: reserved.record.responseId }; + await post(`${route}/bind`).send(body).expect(200); + await post(`${route}/bind`).send(body).expect(200); + const run = await storage.read(tenantId, 'channel', admitted.body.runId); + expect(run?.execution).toMatchObject({ + protocol: 'responses', + responseId: reserved.record.responseId, + conversationId: 'canonical-conversation', + }); + await storage.saveSnapshot( + tenantId, + claimed.body.fence, + { + responseId: reserved.record.responseId, + status: 'requires_action', + text: '', + elapsedMs: 1, + pendingAction: { actionId: 'oauth-action', kind: 'oauth', expiresAt: Date.now() + 60_000 }, + }, + Date.now(), + ); + const reviewPath = `runs/${admitted.body.runId}/review`; + const review = await post(reviewPath) + .send({ actor: event().actor, revision: 'revision-1' }) + .expect(200); + expect(review.body.url).toBe( + `http://127.0.0.1:3088/base/responses/${reserved.record.responseId}/review`, + ); + await post(reviewPath) + .send({ actor: event('U2').actor, revision: 'revision-1' }) + .expect(403); + expect(invoked).toHaveLength(0); + }), +); + +it( + 'holds cancelled unbound credentials and recovers a late canonical response without execution', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const route = `runs/${admitted.body.runId}/response`; + await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + await storage.attachResponseCredential( + tenantId, + claimed.body.fence, + { apiKeyId: 'issued-key', encryptedApiKey: 'ciphertext' }, + Date.now(), + ); + await post(`runs/${admitted.body.runId}/cancel`) + .send({ actor: event().actor, revision: 'revision-1' }) + .expect(202); + await post(`${route}/prepare`) + .send({ fence: claimed.body.fence }) + .expect(409, { error: 'response_cancellation_pending' }); + expect(responseCredential).not.toHaveBeenCalled(); + const reserved = await createResponseStorage(mongoose).reserve({ + tenantId, + userId: users[0].id, + responseId: `resp_${admitted.body.runId}`, + requestKey: `channel:${admitted.body.runId}`, + sourceKeyId: 'issued-key', + fingerprint: 'late-request', + agentId: 'agent', + requestJSON: JSON.stringify({ + model: 'agent', + input: event().input, + background: true, + store: true, + }), + conversationId: 'late-canonical-conversation', + parentMessageId: '00000000-0000-0000-0000-000000000000', + }); + if (!reserved.ok) throw new Error('Reservation failed'); + for (let attempt = 0; attempt < 2; attempt++) { + const prepared = await post(`${route}/prepare`) + .send({ fence: claimed.body.fence }) + .expect(200); + expect(prepared.body.responseId).toBe(reserved.record.responseId); + } + expect(responseCredential).toHaveBeenCalledTimes(2); + expect(responseCredential.mock.calls[0][0]).toMatchObject({ cancellationRequested: true }); + expect(invoked).toHaveLength(0); + expect( + (await storage.read(tenantId, 'channel', admitted.body.runId))?.snapshot, + ).toBeUndefined(); + expect( + await createResponseStorage(mongoose).read(reserved.record, reserved.record.responseId), + ).toEqual(reserved.record); + }), +); + +it.each(['per_user', 'shared'] as const)( + 'keeps native review private and actor-bound in %s mode, including same-account thread participants', + (mode) => + tenantStorage.run({ tenantId }, async () => { + if (mode === 'shared') + await models.AgentChannel.updateOne( + { channelId: 'channel' }, + { $set: { credentialMode: 'shared' } }, + ); + const linked = await link('U1', users[0]); + await link('U2', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const base = `runs/${admitted.body.runId}`; + const prepared = await post(`${base}/response/prepare`) + .send({ fence: claimed.body.fence }) + .expect(200); + const responseId = `resp_${admitted.body.runId}`; + await createResponseStorage(mongoose).reserve({ + tenantId, + userId: users[0].id, + responseId, + requestKey: prepared.body.requestKey, + fingerprint: 'request', + agentId: 'agent', + requestJSON: JSON.stringify(prepared.body.request), + conversationId: 'native-conversation', + parentMessageId: 'root', + }); + await post(`${base}/response/bind`) + .send({ fence: claimed.body.fence, responseId }) + .expect(200); + await storage.saveSnapshot( + tenantId, + claimed.body.fence, + { + responseId, + status: 'requires_action', + text: '', + elapsedMs: 1, + pendingAction: { + actionId: 'native-action', + kind: 'ask_user_question', + expiresAt: Date.now() + 60000, + }, + }, + Date.now(), + ); + const body = { actor: event().actor, revision: 'revision-1', externalChannelId: 'C1' }; + const view = await post(`${base}/review`).send(body).expect(200); + expect(view.body.pendingAction.payload.question.question).toBe('Private question'); + const savedView = await interactive.readInteractiveReview( + { userId: users[0].id, tenantId }, + responseId, + ); + if (!savedView) throw new Error('Expected the current actor review'); + const pendingDecision = { + intentId: 'saved-native-intent', + decision: { actionId: 'native-action', answer: 'Private saved answer' }, + }; + jest.mocked(interactive.readInteractiveReview).mockImplementation(async () => ({ + ...savedView, + pendingDecision, + })); + const saved = await post(`${base}/review`).send(body).expect(200); + expect(saved.body.pendingDecision).toEqual(pendingDecision); + const browser = await post(`${base}/review`) + .send({ actor: event().actor, revision: 'revision-1' }) + .expect(200); + expect(browser.body.pendingAction).toBeUndefined(); + expect(browser.body.pendingDecision).toBeUndefined(); + for (const invalid of [ + { ...body, actor: event('U2').actor }, + { ...body, externalChannelId: 'C2' }, + { ...body, revision: 'stale' }, + ]) { + await post(`${base}/review`).send(invalid).expect(403); + await post(`${base}/review/decision`) + .send({ ...invalid, decision: { actionId: 'native-action', answer: 'yes' } }) + .expect(403); + } + expect(interactive.submitInteractiveReview).not.toHaveBeenCalled(); + await post(`${base}/review/decision`) + .send({ ...body, decision: { actionId: 'native-action', answer: 'yes' } }) + .expect(202); + expect(interactive.submitInteractiveReview).toHaveBeenCalledWith( + { userId: users[0].id, tenantId }, + responseId, + { actionId: 'native-action', answer: 'yes' }, + ); + value( + await authority.identity.revoke( + { tenantId, userId: users[0].id, accountLabel: 'A' }, + linked.linkId, + linked.revision, + ), + ); + await post(`${base}/review`).send(body).expect(403); + await post(`${base}/review/decision`) + .send({ ...body, decision: { actionId: 'native-action', answer: 'yes' } }) + .expect(403); + expect(interactive.submitInteractiveReview).toHaveBeenCalledTimes(1); + expect(invoked).toHaveLength(0); + }), +); + +it( + 'native Stop resolves only the current actor lane and ignores stale or foreign clicks', + scoped(async () => { + await link('U1', users[0]); + await link('U2', users[1]); + const admission = event(); + const admitted = await post('runs').send(admission).expect(201); + await post('runs/claim').send({ owner: 'worker' }).expect(200); + const stop = { + actor: admission.actor, + externalThreadKey: admission.externalThreadKey, + externalChannelId: admission.externalChannelId, + eventAt: Date.now(), + }; + const route = 'runs/stop-thread'; + await post(route) + .send({ ...stop, actor: { ...stop.actor, externalUserId: 'U2' } }) + .expect(200, { cancellationRequested: false }); + await post(route) + .send({ ...stop, externalThreadKey: 'C1:other' }) + .expect(200, { cancellationRequested: false }); + await post(route) + .send({ ...stop, eventAt: 1 }) + .expect(200, { cancellationRequested: false }); + await post(route) + .send({ ...stop, actor: { ...stop.actor, teamId: 'other' } }) + .expect(403); + expect( + (await storage.read(tenantId, 'channel', admitted.body.runId))?.cancellationRequested, + ).toBe(false); + await post(route).send(stop).expect(202, { cancellationRequested: true }); + await post(route).send(stop).expect(202, { cancellationRequested: true }); + expect( + (await storage.read(tenantId, 'channel', admitted.body.runId))?.cancellationRequested, + ).toBe(true); + }), +); + +it( + 'summarizes only terminal snapshots and freezes the first summary across advance retries', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const route = `runs/${admitted.body.runId}/advance`; + const body = { fence: claimed.body.fence }; + await post(route).send(body).expect(200); + expect(summarize).not.toHaveBeenCalled(); + nextSnapshot = { status: 'completed', text: 'Answer', elapsedMs: 12 }; + const first = await post(route).send(body).expect(200); + expect(summarize).toHaveBeenCalledTimes(2); + expect(first.body.snapshot.summary).toMatchObject({ version: 1, throughSequence: 1 }); + summarize.mockRejectedValue(new Error('Optional diagnostics unavailable')); + const replay = await post(route).send(body).expect(200); + expect(replay.body.snapshot.summary).toEqual(first.body.snapshot.summary); + expect(summarize).toHaveBeenCalledTimes(3); + }), +); + +it( + 'keeps a valid terminal answer when optional summary storage is unavailable', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + nextSnapshot = { status: 'completed', text: 'Answer', elapsedMs: 12 }; + summarize.mockRejectedValue(new Error('Private database failure')); + const response = await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(200); + expect(response.body.snapshot).toMatchObject({ status: 'completed', text: 'Answer' }); + expect(response.body.snapshot).not.toHaveProperty('summary'); + expect(JSON.stringify(response.body)).not.toContain('Private database failure'); + }), +); + +it( + 'rechecks actor authority after summary retrieval before persisting or returning the answer', + scoped(async () => { + const linked = await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + nextSnapshot = { status: 'completed', text: 'Do not deliver', elapsedMs: 12 }; + summarize.mockImplementation(async () => { + value( + await authority.identity.revoke( + { tenantId, userId: users[0].id, accountLabel: 'A' }, + linked.linkId, + linked.revision, + ), + ); + throw new Error('Optional metadata unavailable after revocation'); + }); + const response = await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(403); + expect(response.body).toEqual({ error: 'channel_authority_changed' }); + const persisted = await storage.read(tenantId, 'channel', admitted.body.runId); + expect(persisted?.state).toBe('cancelled'); + expect(persisted?.snapshot).toBeUndefined(); + expect(retired).toHaveLength(1); + }), +); + +it( + 'does not swallow an authority lookup outage as an optional summary failure', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + nextSnapshot = { status: 'completed', text: 'Do not deliver', elapsedMs: 12 }; + const lookup = jest.spyOn(db, 'findUser'); + summarize.mockImplementation(async () => { + lookup.mockRejectedValue(new Error('Authority storage unavailable')); + throw new Error('Optional summary unavailable'); + }); + try { + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(503); + const persisted = await storage.read(tenantId, 'channel', admitted.body.runId); + expect(persisted?.state).toBe('executing'); + expect(persisted?.snapshot).toBeUndefined(); + expect(retired).toHaveLength(0); + } finally { + lookup.mockRestore(); + } + }), +); + +it( + 'reports only a stable execution hold code without exposing diagnostics or fabricating completion', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker-A' }).expect(200); + advanceError = new ChannelExecutionError('dispatch_outcome_unknown'); + advanceError.message = 'sensitive provider details must remain private'; + const response = await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(409); + expect(response.body).toEqual({ error: 'dispatch_outcome_unknown' }); + const persisted = await storage.read(tenantId, 'channel', admitted.body.runId); + expect(persisted?.state).toBe('executing'); + expect(persisted?.snapshot).toBeUndefined(); + expect(invoked).toHaveLength(0); + }), +); + +it( + 'validates and durably preserves opaque provider presentation with an immutable text fallback', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker-A' }).expect(200); + const fence = claimed.body.fence; + await post(`runs/${admitted.body.runId}/advance`).send({ fence }).expect(200); + await bindFixtureResponse( + (await storage.read(tenantId, 'channel', admitted.body.runId))!, + 'reply-1', + ); + await storage.saveSnapshot( + tenantId, + fence, + { status: 'completed', responseId: 'reply-1', text: '# Answer', elapsedMs: 1 }, + Date.now(), + ); + const payload = JSON.stringify([ + { type: 'header', text: { type: 'plain_text', text: 'Answer' } }, + ]); + const part = { + operationId: 'reply-part', + kind: 'message', + required: true, + text: '*Answer*', + presentation: { format: 'slack-blocks-v1', payload }, + }; + const plan = { revision: 'plan-1', rendererVersion: 'slack-structured-v2', parts: [part] }; + const route = `runs/${admitted.body.runId}/plan`; + for (const invalid of [ + { ...part, kind: 'artifact' }, + { ...part, presentation: { format: 'bad format', payload } }, + { ...part, presentation: { format: 'slack-blocks-v1', payload: '{' } }, + { ...part, presentation: { format: 'slack-blocks-v1', payload: 'null' } }, + { + ...part, + presentation: { format: 'slack-blocks-v1', payload: JSON.stringify(['界'.repeat(22_000)]) }, + }, + ]) { + await post(route) + .send({ fence, plan: { ...plan, parts: [invalid] } }) + .expect(400); + } + await post(route).send({ fence, plan }).expect(200); + expect((await storage.read(tenantId, 'channel', admitted.body.runId))?.plan).toEqual(plan); + await post(route).send({ fence, plan }).expect(200); + await post(route) + .send({ + fence, + plan: { + ...plan, + parts: [{ ...part, presentation: { ...part.presentation, payload: '[]' } }], + }, + }) + .expect(409); + }), +); +function post(path: string) { + return request(app) + .post(`/service/channel/${path}`) + .set('Authorization', `Bearer ${token}`) + .set('X-Tenant-Id', tenantId); +} + +it( + 'yields only a currently authorized exact polling lease and rejects revoked authority', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + await models.ChannelRun.updateOne( + { tenantId, runId: run.runId }, + { + $set: { execution: { ...run.execution, protocol: 'responses', responseId: 'response-1' } }, + }, + ); + const route = `runs/${admitted.body.runId}/yield`; + await request(app) + .post(`/service/channel/${route}`) + .send({ fence: claimed.body.fence }) + .expect(401); + await post(route) + .send({ fence: { ...claimed.body.fence, runId: 'foreign' } }) + .expect(400); + await post(route) + .send({ fence: { ...claimed.body.fence, owner: 'foreign' } }) + .expect(409); + const yieldSpy = jest.spyOn(storage, 'yieldLease'); + acl.set(users[0].id, 0); + await post(route).send({ fence: claimed.body.fence }).expect(403); + expect(yieldSpy).not.toHaveBeenCalled(); + yieldSpy.mockRestore(); + }), +); + +it( + 'acknowledges a safe polling yield without invoking an Agent or releasing its lane', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + await models.ChannelRun.updateOne( + { tenantId, runId: run.runId }, + { + $set: { execution: { ...run.execution, protocol: 'responses', responseId: 'response-1' } }, + }, + ); + const route = `runs/${admitted.body.runId}/yield`; + await post(route).send({ fence: claimed.body.fence }).expect(204); + await post(route).send({ fence: claimed.body.fence }).expect(409); + expect((await storage.read(tenantId, 'channel', run.runId))?.leaseOwner).toBeUndefined(); + expect(invoked).toHaveLength(0); + const lane = await models.ChannelLane.findOne({ tenantId, laneId: run.laneId }).lean(); + expect(lane?.activeRunId).toBe(run.runId); + }), +); +async function link(actor: string, user: IUser) { + const channel = (await db.getAgentChannel('channel'))!; + const identity = channelExternalIdentity(channel, actor); + const issued = value(await authority.identity.issue(identity)); + const ticket = new URL(issued.url).hash.slice(1); + const principal = { tenantId, userId: user.id, accountLabel: user.name! }; + const preview = value(await authority.identity.preview(principal, ticket)); + const accepted = value(await authority.identity.accept(principal, ticket, preview.revision)); + return value(await authority.identity.confirm(identity, accepted.linkId, accepted.revision)); +} + +it( + 'routes same-thread A and B to their own linked principals and independent histories through actual service handlers', + scoped(async () => { + await link('U1', users[0]); + await post('runs').send(event('U2')).expect(403); + await link('U2', users[1]); + const a = await post('runs').send(event()).expect(201); + const b = await post('runs').send(event('U2')).expect(201); + const claimedA = await post('runs/claim').send({ owner: 'worker-A' }).expect(200); + await post(`runs/${a.body.runId}/advance`).send({ fence: claimedA.body.fence }).expect(200); + const claimedB = await post('runs/claim').send({ owner: 'worker-B' }).expect(200); + await post(`runs/${b.body.runId}/advance`).send({ fence: claimedB.body.fence }).expect(200); + expect(invoked.map((run) => run.principal.userId)).toEqual([users[0].id, users[1].id]); + expect(invoked[0].execution?.conversationId).not.toBe(invoked[1].execution?.conversationId); + expect(invoked[1].execution?.parentMessageId).toBe('00000000-0000-0000-0000-000000000000'); + expect( + await storage.requestCancel( + { + tenantId, + channelId: 'channel', + agentId: 'agent', + principal: (await authority.resolvePrincipal( + (await db.getAgentChannelMetadata('channel'))!, + event('U2').actor, + ))!, + admission: event('U2'), + }, + a.body.runId, + ), + ).toBe(false); + }), +); + +it( + 'accepts cancel only from the original actor with its current account link and channel revision', + scoped(async () => { + const linked = await link('U1', users[0]); + await link('U2', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const path = `runs/${admitted.body.runId}/cancel`; + await post(path) + .send({ actor: event('U2').actor, revision: 'revision-1' }) + .expect(403); + await post(path).send({ actor: event().actor, revision: 'old' }).expect(403); + expect( + (await storage.read(tenantId, 'channel', admitted.body.runId))?.cancellationRequested, + ).toBe(false); + await post(path).send({ actor: event().actor, revision: 'revision-1' }).expect(202); + expect( + (await storage.read(tenantId, 'channel', admitted.body.runId))?.cancellationRequested, + ).toBe(true); + value( + await authority.identity.revoke( + { tenantId, userId: users[0].id, accountLabel: 'A' }, + linked.linkId, + linked.revision, + ), + ); + await post(path).send({ actor: event().actor, revision: 'revision-1' }).expect(403); + }), +); + +it( + 'shared mode is deny-by-default owner-only and never falls back to creator authority for B', + scoped(async () => { + await db.updateAgentChannel('channel', users[0].id, { credentialMode: 'shared' }); + await link('U1', users[0]); + await expect(link('U2', users[1])).rejects.toThrow('forbidden'); + await post('runs').send(event('U2')).expect(403); + await post('runs') + .send({ ...event('U2'), principal: { userId: users[0].id } }) + .expect(400); + await post('runs').send(event()).expect(201); + }), +); + +it( + 'rechecks link revocation and current ACL before execution, not only at admission', + scoped(async () => { + const linked = await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + acl.set(users[0].id, 0); + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(403); + expect((await storage.read(tenantId, 'channel', admitted.body.runId))?.state).toBe('cancelled'); + acl.set(users[0].id, PermissionBits.VIEW); + const next = await post('runs').send(event('U1', 'after-acl-restored')).expect(201); + const nextClaim = await post('runs/claim').send({ owner: 'worker' }).expect(200); + value( + await authority.identity.revoke( + { tenantId, userId: users[0].id, accountLabel: 'A' }, + linked.linkId, + linked.revision, + ), + ); + await post(`runs/${next.body.runId}/advance`).send({ fence: nextClaim.body.fence }).expect(403); + expect(invoked).toHaveLength(0); + }), +); + +it( + 'retires revoked queued work without exposing its view, then admits the next actor lane', + scoped(async () => { + const linked = await link('U1', users[0]); + await link('U2', users[1]); + const a = await post('runs').send(event()).expect(201); + const b = await post('runs').send(event('U2')).expect(201); + value( + await authority.identity.revoke( + { tenantId, userId: users[0].id, accountLabel: 'A' }, + linked.linkId, + linked.revision, + ), + ); + await post('runs/claim').send({ owner: 'cleanup-worker' }).expect(204); + expect(retired).toHaveLength(1); + expect(retired[0]).toMatchObject({ + runId: a.body.runId, + cancellationRequested: true, + errorCode: 'channel_authority_changed', + }); + expect(retired[0].execution).toBeUndefined(); + expect((await storage.read(tenantId, 'channel', a.body.runId))?.state).toBe('cancelled'); + const next = await post('runs/claim').send({ owner: 'next-worker' }).expect(200); + expect(next.body.run.runId).toBe(b.body.runId); + expect(invoked).toHaveLength(0); + }), +); + +it( + 'requires the exact worker fence for cleanup, never another actor action, and latches authority loss', + scoped(async () => { + await link('U1', users[0]); + await link('U2', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + acl.set(users[0].id, 0); + await post(`runs/${admitted.body.runId}/cancel`) + .send({ actor: event('U2').actor, revision: 'revision-1' }) + .expect(403); + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: { ...claimed.body.fence, owner: 'other-worker' } }) + .expect(409); + expect(retired).toHaveLength(0); + retirement = 'pending'; + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(403); + expect((await storage.read(tenantId, 'channel', admitted.body.runId))?.state).toBe('queued'); + acl.set(users[0].id, PermissionBits.VIEW); + retirement = 'stopped'; + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(403); + expect((await storage.read(tenantId, 'channel', admitted.body.runId))?.state).toBe('cancelled'); + expect(invoked).toHaveLength(0); + }), +); + +it( + 'does not turn a permission lookup outage into revocation or cleanup', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const lookup = jest + .spyOn(db, 'getAgentChannel') + .mockRejectedValueOnce(new Error('database unavailable')); + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(503); + lookup.mockRestore(); + expect(retired).toHaveLength(0); + expect(await storage.read(tenantId, 'channel', admitted.body.runId)).toMatchObject({ + cancellationRequested: false, + state: 'queued', + }); + await post(`runs/${admitted.body.runId}/advance`) + .send({ fence: claimed.body.fence }) + .expect(200); + }), +); + +it( + 'rejects wrong workspace/app, foreign tenant and unauthenticated admission', + scoped(async () => { + await link('U1', users[0]); + const forged = event(); + forged.actor.teamId = 'T2'; + await post('runs').send(forged).expect(403); + await request(app).post('/service/channel/runs').send(event()).expect(401); + await request(app) + .post('/service/channel/runs') + .set('Authorization', `Bearer ${token}`) + .set('X-Tenant-Id', 'other') + .send(event()) + .expect(404); + expect(invoked).toHaveLength(0); + }), +); + +it.each(['revoked', 'renewed', 'expired', 'replaced'] as const)( + 'checks refreshed authority and lease after artifact retrieval: %s', + (change) => + scoped(async () => { + const linked = await link('U1', users[0]); + await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const fence = claimed.body.fence; + const run = await storage.prepareExecution(tenantId, fence, Date.now()); + await bindFixtureResponse(run, 'response'); + const file = await db.publishRunArtifactFile({ + scope: { + userId: users[0].id, + tenantId, + conversationId: run.execution!.conversationId, + runId: 'response', + executionId: 'execution', + agentId: 'agent', + sourceFileId: 'source', + }, + provenance: { + runId: 'response', + executionId: 'execution', + agentId: 'agent', + sourceFileId: 'source', + publishedAt: new Date().toISOString(), + inputFileIds: [], + }, + file: { + filename: 'result.txt', + filepath: '/synthetic/result.txt', + bytes: 4, + type: 'text/plain', + source: FileSources.local, + }, + }); + const sha256 = createHash('sha256').update('file').digest('hex'); + const descriptor = { + artifactId: `${file.file_id}:${sha256}`, + filename: 'result.txt', + mediaType: 'text/plain', + bytes: 4, + sha256, + }; + await storage.saveSnapshot( + tenantId, + fence, + { + status: 'completed', + responseId: 'response', + text: 'answer', + elapsedMs: 1, + artifacts: [descriptor], + }, + Date.now(), + ); + const initial = (await storage.read(tenantId, 'channel', run.runId))!; + artifactRead = async () => { + if (change === 'revoked') { + value( + await authority.identity.revoke( + { tenantId, userId: users[0].id, accountLabel: 'A' }, + linked.linkId, + linked.revision, + ), + ); + return; + } + if (change === 'renewed') + await storage.renew(tenantId, fence, initial.leaseExpiresAt - 1, 60000); + fakeNow = initial.leaseExpiresAt + 1; + if (change === 'replaced') + expect( + await storage.claim(tenantId, 'channel', 'replacement', fakeNow, 60000), + ).not.toBeNull(); + }; + const response = await post( + `runs/${run.runId}/artifacts/${encodeURIComponent(descriptor.artifactId)}`, + ) + .send({ fence }) + .expect({ renewed: 200, revoked: 403, expired: 409, replaced: 409 }[change]); + if (change === 'renewed') { + expect(response.body).toEqual(Buffer.from('file')); + expect(response.headers['x-artifact-sha256']).toBe(sha256); + } else { + expect(response.body).toEqual({ + error: change === 'revoked' ? 'channel_authority_changed' : 'stale_fence', + }); + expect(response.headers['x-artifact-sha256']).toBeUndefined(); + if (change === 'revoked') + expect((await storage.read(tenantId, 'channel', run.runId))?.state).toBe('cancelled'); + } + })(), +); + +it.each(['expired', 'revoked', 'changed-binding'] as const)( + 'refuses %s issued credentials without minting replacements', + (change) => + scoped(async () => { + await link('U1', users[0]); + await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + const credentials = responseCredentials(); + await credentials.provision(run); + const saved = (await storage.read(tenantId, 'channel', run.runId))!; + expect(await credentials.isCurrent(saved)).toBe(true); + let checked = saved; + if (change === 'changed-binding') { + const foreign = await db.createAgentApiKey({ + userId: users[0].id, + name: 'Other run credential', + binding: { kind: 'channel_run', id: 'another/run' }, + }); + checked = { ...saved, execution: { ...saved.execution!, apiKeyId: foreign.id } }; + } else if (change === 'revoked') + await db.deleteAgentApiKey(saved.execution!.apiKeyId!, users[0].id); + else + await models.AgentApiKey.updateOne( + { _id: saved.execution!.apiKeyId }, + { $set: { expiresAt: new Date(0) } }, + ); + const create = jest.spyOn(db, 'createAgentApiKey'); + expect(await credentials.isCurrent(checked)).toBe(false); + await expect(credentials.provision(checked)).rejects.toThrow('authority changed'); + expect(create).not.toHaveBeenCalled(); + expect((await storage.read(tenantId, 'channel', run.runId))?.execution?.apiKeyId).toBe( + saved.execution!.apiKeyId, + ); + })(), +); + +it( + 'reauthorizes trace operations on retry without rewriting frozen answers or metric totals', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claim = await post('runs/claim').send({ owner: 'first' }).expect(200); + const run = (await storage.read(tenantId, 'channel', admitted.body.runId))!; + const metrics = await summarize(run, { status: 'completed', text: 'Answer', elapsedMs: 12 }); + let allowed = true; + const traceUrl = 'https://cloud.langfuse.com/project/project/sessions/conversation'; + let currentUrl = traceUrl; + let amount = '1'; + summarize.mockImplementation(async () => ({ + ...metrics, + costUSD: { coverage: 'complete', coveredRuns: 1, value: amount, basis: 'provider_reported' }, + ...(allowed ? { session: { url: currentUrl } } : {}), + })); + nextSnapshot = { status: 'completed', text: 'Answer', elapsedMs: 12 }; + const route = `runs/${admitted.body.runId}`; + const first = await post(`${route}/advance`).send({ fence: claim.body.fence }).expect(200); + const answer = { operationId: 'answer', kind: 'message', required: true, text: 'Answer' }; + const trace = { + operationId: 'trace', + kind: 'message', + required: false, + text: traceUrl, + traceUrl, + }; + const plan = { revision: 'v1', rendererVersion: 'v1', parts: [answer, trace] }; + await post(`${route}/plan`) + .send({ + fence: claim.body.fence, + plan: { ...plan, parts: [answer, { ...trace, required: true }] }, + }) + .expect(400); + const planned = await post(`${route}/plan`).send({ fence: claim.body.fence, plan }).expect(200); + expect(planned.body.plan.parts).toEqual(plan.parts); + allowed = false; + amount = '999'; + const replay = await post(`${route}/advance`).send({ fence: claim.body.fence }).expect(200); + expect(JSON.stringify(replay.body)).not.toContain(traceUrl); + expect(replay.body.plan.parts).toEqual([answer]); + expect(replay.body.snapshot.summary.costUSD).toEqual(first.body.snapshot.summary.costUSD); + fakeNow = Date.now() + 60001; + const retry = await post('runs/claim').send({ owner: 'replacement' }).expect(200); + expect(JSON.stringify(retry.body)).not.toContain(traceUrl); + await post(`${route}/begin`) + .send({ fence: retry.body.fence, operationId: 'trace' }) + .expect(409, { error: 'trace_unavailable' }); + expect( + (await storage.read(tenantId, 'channel', admitted.body.runId))?.receipts.find( + (receipt) => receipt.operationId === 'trace', + ), + ).toMatchObject({ attempts: 0, outcome: 'planned' }); + await post(`${route}/begin`) + .send({ fence: retry.body.fence, operationId: 'answer' }) + .expect(200); + allowed = true; + currentUrl = 'https://cloud.langfuse.com/project/changed/sessions/conversation'; + const changed = await post(`${route}/advance`).send({ fence: retry.body.fence }).expect(200); + expect(changed.body.snapshot.summary.session.url).toBe(currentUrl); + expect(changed.body.plan.parts).toEqual([answer]); + expect(JSON.stringify(changed.body)).not.toContain(traceUrl); + const retained = (await storage.read(tenantId, 'channel', admitted.body.runId))!; + expect(retained.plan).toEqual(plan); + expect(retained.snapshot!.summary!.costUSD).toEqual(first.body.snapshot.summary.costUSD); + }), +); + +it.each(['revoked', 'expired', 'foreign-binding'] as const)( + 'rejects %s keys in both source and request authorization', + (change) => + scoped(async () => { + await link('U1', users[0]); + await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + const credentials = responseCredentials(); + await credentials.provision(run); + const saved = (await storage.read(tenantId, 'channel', run.runId))!; + const binding = { kind: 'channel_run' as const, id: `channel/${run.runId}` }; + const keyId = saved.execution!.apiKeyId!; + const owner = { tenantId, userId: users[0].id }; + const probe = express(); + probe.use(express.json()); + probe.post('/probe', async (req, res) => { + req.originalUrl = '/api/agents/v1/responses'; + res.json({ + allowed: await credentials.authorizeRequest({ + keyId, + binding, + user: users[0], + request: req, + }), + }); + }); + const requestAllowed = async () => + ( + await request(probe) + .post('/probe') + .set('Idempotency-Key', `channel:${run.runId}`) + .send({ model: 'agent', input: run.admission.input, background: true, store: true }) + .expect(200) + ).body.allowed; + expect(await credentials.authorizeSource(binding, keyId, owner, 'agent')).toBe(true); + expect(await requestAllowed()).toBe(true); + if (change === 'revoked') await db.deleteAgentApiKey(keyId, users[0].id); + else + await models.AgentApiKey.collection.updateOne( + { _id: new mongoose.Types.ObjectId(keyId) }, + { + $set: + change === 'expired' + ? { expiresAt: new Date(0) } + : { binding: { kind: 'channel_run', id: 'other/run' } }, + }, + ); + expect(await credentials.authorizeSource(binding, keyId, owner, 'agent')).toBe(false); + expect(await requestAllowed()).toBe(false); + })(), +); + +it( + 'requires persisted cancellation intent for a run-key cancel capability', + scoped(async () => { + await link('U1', users[0]); + await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + const credentials = responseCredentials(); + await credentials.provision(run); + const saved = (await storage.read(tenantId, 'channel', run.runId))!; + await bindFixtureResponse(saved, 'resp_cancel'); + const probe = express(); + probe.post('/probe', async (req, res) => { + req.originalUrl = '/api/agents/v1/responses/resp_cancel/cancel'; + res.json({ + allowed: await credentials.authorizeRequest({ + keyId: saved.execution!.apiKeyId!, + binding: { kind: 'channel_run', id: `channel/${run.runId}` }, + user: users[0], + request: req, + }), + }); + }); + expect((await request(probe).post('/probe').expect(200)).body.allowed).toBe(false); + await storage.requestCancel(saved, saved.runId); + expect((await request(probe).post('/probe').expect(200)).body.allowed).toBe(true); + }), +); + +it.each([false, true])( + 'starts independent authorization reads together and evaluates final expiry: %s', + (expires) => + scoped(async () => { + await link('U1', users[0]); + await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + await responseCredentials().provision(run); + const saved = (await storage.read(tenantId, 'channel', run.runId))!; + const key = (await db.getAgentApiKeyById(saved.execution!.apiKeyId!, users[0].id))!; + let clock = Date.now(); + key.expiresAt = new Date(clock + 10); + let entered!: () => void; + let release!: () => void; + const policyEntered = new Promise((resolve) => { + entered = resolve; + }); + const policyGate = new Promise((resolve) => { + release = resolve; + }); + const readKey = jest.fn(async () => key); + const credentials = responseCredentials({ + now: () => clock, + getKey: readKey, + resolvePrincipal: async () => { + entered(); + await policyGate; + return saved.principal; + }, + }); + const result = credentials.authorizeSource( + { kind: 'channel_run', id: `channel/${run.runId}` }, + saved.execution!.apiKeyId!, + { tenantId, userId: users[0].id }, + 'agent', + ); + try { + await policyEntered; + expect(readKey).toHaveBeenCalledTimes(1); + if (expires) clock += 20; + release(); + expect(await result).toBe(!expires); + } finally { + release(); + await result.catch(() => undefined); + } + })(), +); + +it( + 'reconciles verified unknown delivery once and releases the actor lane without reexecution', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const first = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const next = await post('runs').send(event('U1', 'next')).expect(201); + const route = `runs/${admitted.body.runId}`; + nextSnapshot = { status: 'completed', text: 'Answer', elapsedMs: 1 }; + await post(`${route}/advance`).send({ fence: first.body.fence }).expect(200); + await post(`${route}/plan`) + .send({ + fence: first.body.fence, + plan: { + revision: 'plan', + rendererVersion: 'test', + parts: [{ operationId: 'answer', kind: 'message', required: true, text: 'Answer' }], + }, + }) + .expect(200); + await post(`${route}/begin`) + .send({ fence: first.body.fence, operationId: 'answer' }) + .expect(200); + const receipt = { operationId: 'answer', outcome: 'unknown', attempts: 1 }; + await post(`${route}/settle`).send({ fence: first.body.fence, receipt }).expect(200); + await post(`${route}/commit`) + .send({ fence: first.body.fence }) + .expect(409, { error: 'delivery_incomplete' }); + await post(`${route}/begin`) + .send({ fence: first.body.fence, operationId: 'answer' }) + .expect(409); + fakeNow = (await storage.read(tenantId, 'channel', admitted.body.runId))!.leaseExpiresAt + 1; + const replacement = await post('runs/claim').send({ owner: 'reconciler' }).expect(200); + const fence = replacement.body.fence; + const verified = { ...receipt, outcome: 'confirmed', externalMessageKey: 'C1:verified' }; + await post(`${route}/delivery/reconcile`) + .send({ fence: first.body.fence, receipt: verified }) + .expect(409); + await post(`${route}/delivery/reconcile`) + .send({ fence, receipt: { ...verified, attempts: 2 } }) + .expect(409); + await post(`${route}/delivery/reconcile`) + .send({ fence, receipt: { ...verified, externalMessageKey: 'C2:foreign' } }) + .expect(400); + await post(`${route}/delivery/reconcile`) + .send({ fence, receipt: { ...receipt, outcome: 'confirmed' } }) + .expect(409); + await post(`${route}/delivery/reconcile`).send({ fence, receipt: verified }).expect(200); + await post(`${route}/delivery/reconcile`).send({ fence, receipt: verified }).expect(200); + await post(`${route}/delivery/reconcile`) + .send({ fence, receipt: { ...verified, externalMessageKey: 'C1:changed' } }) + .expect(409); + const delivered = await post(`${route}/commit`).send({ fence }).expect(200); + expect(delivered.body.state).toBe('delivered'); + expect(delivered.body.receipts[0]).toMatchObject({ + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C1:verified', + }); + expect(invoked).toHaveLength(1); + const nextClaim = await post('runs/claim').send({ owner: 'next-worker' }).expect(200); + expect(nextClaim.body.run.runId).toBe(next.body.runId); + }), +); + +it( + 'reconciles verified unknown progress with the same revision and provider identity', + scoped(async () => { + await link('U1', users[0]); + const admitted = await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const fence = claimed.body.fence; + const route = `runs/${admitted.body.runId}/progress`; + await post(`${route}/plan`) + .send({ fence, revision: 1, rendererVersion: 'test', text: 'Working' }) + .expect(200); + await post(`${route}/begin`).send({ fence, revision: 1 }).expect(200); + await post(`${route}/settle`) + .send({ fence, revision: 1, receipt: { outcome: 'unknown', attempts: 1 } }) + .expect(200); + const receipt = { outcome: 'confirmed', attempts: 1, externalMessageKey: 'C1:progress' }; + await post(`${route}/reconcile`).send({ fence, revision: 2, receipt }).expect(409); + await post(`${route}/reconcile`) + .send({ fence, revision: 1, receipt: { ...receipt, externalMessageKey: 'C2:foreign' } }) + .expect(400); + await post(`${route}/reconcile`).send({ fence, revision: 1, receipt }).expect(200); + await post(`${route}/reconcile`).send({ fence, revision: 1, receipt }).expect(200); + await post(`${route}/reconcile`) + .send({ fence, revision: 1, receipt: { ...receipt, externalMessageKey: 'C1:changed' } }) + .expect(409); + const updated = await post(`${route}/plan`) + .send({ fence, revision: 2, rendererVersion: 'test', text: 'Updated' }) + .expect(200); + expect(updated.body.progress).toMatchObject({ + revision: 2, + text: 'Updated', + externalMessageKey: 'C1:progress', + }); + }), +); + +it( + 'propagates key lookup outages without converting them to revocation', + scoped(async () => { + await link('U1', users[0]); + await post('runs').send(event()).expect(201); + const claimed = await post('runs/claim').send({ owner: 'worker' }).expect(200); + const run = await storage.prepareExecution(tenantId, claimed.body.fence, Date.now()); + await responseCredentials().provision(run); + const saved = (await storage.read(tenantId, 'channel', run.runId))!; + const credentials = responseCredentials({ + getKey: async () => { + throw new Error('key store unavailable'); + }, + }); + await expect( + credentials.authorizeSource( + { kind: 'channel_run', id: `channel/${run.runId}` }, + saved.execution!.apiKeyId!, + { tenantId, userId: users[0].id }, + 'agent', + ), + ).rejects.toThrow('key store unavailable'); + expect((await storage.read(tenantId, 'channel', run.runId))?.cancellationRequested).toBe(false); + }), +); diff --git a/packages/api/src/channels/cleanup.spec.ts b/packages/api/src/channels/cleanup.spec.ts new file mode 100644 index 0000000000..333777d790 --- /dev/null +++ b/packages/api/src/channels/cleanup.spec.ts @@ -0,0 +1,629 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { channelConfigSchema } from 'librechat-data-provider'; +import { + createModels, + createMethods, + createChannelRunStorage, + createResponseStorage, + runAsSystem, + tenantStorage, +} from '@librechat/data-schemas'; +import type { ChannelRunRecord } from '@librechat/data-schemas'; +import type { ChannelCleanupDependencies } from './cleanup'; +import { createResponseRetirement } from '~/agents/responses/retirement'; +import { createChannelResponseRetirement } from './responses'; +import { createChannelExecutionAdapter } from './execution'; +import { createChannelCleanupRuntime } from './cleanup'; +import * as recovery from '~/agents/recovery'; + +let mongo: MongoMemoryReplSet; +let db: ReturnType; +let storage: ReturnType; +let deps: ChannelCleanupDependencies; +let runtime: ReturnType; +let retired: ChannelRunRecord[]; +let allowed: boolean; +let enabled: boolean; +let stoppedResult: 'stopped' | 'pending'; +let clock: number; +const tenantId = 'cleanup-tenant'; +const scope = (work: () => Promise) => tenantStorage.run({ tenantId }, work); +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await mongoose.connect(mongo.getUri()); + createModels(mongoose); + db = createMethods(mongoose); + storage = createChannelRunStorage(mongoose); + await Promise.all([mongoose.models.ChannelRun.init(), mongoose.models.ChannelLane.init()]); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(() => + Promise.all( + [ + 'ChannelRun', + 'ChannelLane', + 'AgentChannel', + 'ChannelIdentity', + 'ChannelScope', + 'ResponseScope', + 'StoredResponse', + ].map((name) => mongoose.models[name].deleteMany({})), + ), + ); + clock = Date.now(); + allowed = true; + enabled = true; + retired = []; + stoppedResult = 'stopped'; + await scope(() => + db.createAgentChannel({ + channelId: 'channel', + tenantId, + agentId: 'agent', + createdBy: 'user-a', + type: 'slack', + credentialMode: 'per_user', + status: 'active', + revision: 'v1', + encryptedBotToken: 'not-a-real-token', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' }, + }), + ); + await scope(() => + mongoose.models.ChannelIdentity.create({ + tenantId, + linkId: 'link', + revision: 'link-v1', + state: 'active', + userId: 'user-a', + ticketHash: 'ticket', + ticketExpiresAtMs: clock + 60000, + expiresAtMs: clock + 600000, + identity: { + tenantId, + channelId: 'channel', + channelRevision: 'v1', + teamId: 'T1', + externalUserId: 'U1', + agentId: 'agent', + }, + }), + ); + deps = { + storage, + isCredentialCurrent: async () => true, + channels: db, + now: () => clock, + getConfig: async () => + channelConfigSchema.parse({ + enabled, + cleanup: { intervalMs: 60_000, maxIdleIntervalMs: 60_000, batchSize: 2 }, + }), + resolvePrincipal: async () => + allowed + ? { userId: 'user-a', channelRevision: 'v1', linkId: 'link', linkRevision: 'link-v1' } + : null, + retireForAuthorityLoss: async (run) => { + expect(tenantStorage.getStore()?.tenantId).toBe( + run.tenantId === '_' ? undefined : run.tenantId, + ); + retired.push(run); + return stoppedResult; + }, + deleteApiKey: async () => true, + onError: jest.fn(), + }; + runtime = createChannelCleanupRuntime(deps); +}); +afterEach(async () => { + await runtime.stop(); +}); +async function admit(eventId = 'event') { + return scope(() => + storage.admit( + { + tenantId, + channelId: 'channel', + agentId: 'agent', + principal: { + userId: 'user-a', + channelRevision: 'v1', + linkId: 'link', + linkRevision: 'link-v1', + }, + admission: { + eventId, + externalChannelId: 'C1', + externalThreadKey: 'C1:1', + externalMessageKey: `C1:${eventId}`, + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + input: 'private input', + }, + }, + clock, + ), + ); +} +async function restart() { + await runtime.stop(); + runtime = createChannelCleanupRuntime(deps); + await runtime.initialize(); +} + +it('does not start by construction, initializes once, and leaves authorized active work untouched', async () => { + const { run } = await admit(); + expect(retired).toHaveLength(0); + await Promise.all([runtime.initialize(), runtime.initialize()]); + expect(retired).toHaveLength(0); + expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({ + state: 'queued', + leaseEpoch: 0, + cancellationRequested: false, + }); + await runtime.stop(); + allowed = false; + await runtime.initialize(); + expect(retired).toHaveLength(0); +}); + +it.each(['revoked', 'disabled', 'feature-off', 'orphan'] as const)( + 'cleans %s work without a gateway or any credential bundle', + async (reason) => { + const { run } = await admit(); + await scope(() => storage.claim(tenantId, 'channel', 'absent-gateway', clock, 60_000)); + if (reason === 'revoked') allowed = false; + if (reason === 'disabled') + await scope(() => db.updateAgentChannel('channel', 'user-a', { status: 'disabled' })); + if (reason === 'feature-off') enabled = false; + if (reason === 'orphan') + await scope(() => mongoose.models.AgentChannel.deleteOne({ channelId: 'channel' })); + const bundles = jest.spyOn(db, 'getAgentChannel'); + await runtime.initialize(); + expect(bundles).not.toHaveBeenCalled(); + expect(retired).toHaveLength(1); + expect(retired[0]).toMatchObject({ runId: run.runId, tenantId, principal: run.principal }); + expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({ + state: 'cancelled', + errorCode: 'channel_authority_changed', + }); + expect(deps.onError).not.toHaveBeenCalled(); + }, +); + +it('defers deletion while cleanup is unknown and completes after restart and exact lease expiry', async () => { + const { run } = await admit(); + await scope(() => db.updateAgentChannel('channel', 'user-a', { status: 'deleting' })); + expect(await scope(() => db.deleteAgentChannel('channel', 'user-a'))).toEqual({ + status: 'pending_drain', + }); + stoppedResult = 'pending'; + await runtime.initialize(); + expect(await scope(() => db.getAgentChannelMetadata('channel'))).toMatchObject({ + status: 'deleting', + }); + await restart(); + expect(retired).toHaveLength(1); + clock += 60_001; + stoppedResult = 'stopped'; + await restart(); + expect(await scope(() => db.getAgentChannelMetadata('channel'))).toBeNull(); + expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({ + state: 'cancelled', + }); +}); + +it('recovers deleting parents with no remaining runs but never bypasses a failed legacy key revocation', async () => { + await scope(() => db.updateAgentChannel('channel', 'user-a', { apiKeyId: 'legacy-key-id' })); + await scope(() => db.beginAgentChannelDeletion('channel', 'user-a')); + deps.deleteApiKey = async () => { + throw new Error('key store unavailable'); + }; + await runtime.initialize(); + expect(await scope(() => db.getAgentChannelMetadata('channel'))).toMatchObject({ + status: 'deleting', + }); + deps.deleteApiKey = async () => true; + await restart(); + expect(await scope(() => db.getAgentChannelMetadata('channel'))).toBeNull(); +}); + +it('does not convert authority lookup outages into cleanup', async () => { + const { run } = await admit(); + deps.resolvePrincipal = async () => { + throw new Error('permissions unavailable'); + }; + await runtime.initialize(); + expect(retired).toHaveLength(0); + expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({ + state: 'queued', + cancellationRequested: false, + leaseEpoch: 0, + }); + expect(deps.onError).toHaveBeenCalledTimes(1); +}); + +it('enumerates only bounded metadata in system context then scopes every full read to its tenant', async () => { + await admit('one'); + await admit('two'); + await admit('three'); + const metadata = await runAsSystem(() => storage.listPendingMetadata(undefined, 2)); + expect(metadata.runs).toHaveLength(2); + expect(metadata.next).toBeDefined(); + expect(Object.keys(metadata.runs[0]).sort()).toEqual(['channelId', 'runId', 'tenantId']); + const read = jest.spyOn(storage, 'read'); + allowed = false; + await runtime.initialize(); + expect(retired).toHaveLength(2); + expect(read.mock.calls.every(([tenant]) => tenant === tenantId)).toBe(true); + expect(await scope(() => db.getAgentChannelMetadata('channel'))).not.toHaveProperty( + 'encryptedBotToken', + ); +}); + +it('waits for owned cleanup on shutdown and does not start the next run', async () => { + await admit('one'); + await admit('two'); + allowed = false; + let entered!: () => void; + let release!: () => void; + const ready = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + deps.retireForAuthorityLoss = async (run) => { + retired.push(run); + entered(); + await gate; + return 'stopped'; + }; + const initializing = runtime.initialize(); + await ready; + let ended = false; + const stopping = runtime.stop().then(() => { + ended = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(ended).toBe(false); + release(); + await initializing; + await stopping; + expect(retired).toHaveLength(1); +}); + +it('uses an allowlist so future top-level or Slack credential fields cannot enter authority metadata', async () => { + await mongoose.models.AgentChannel.collection.updateOne( + { tenantId, channelId: 'channel' }, + { $set: { futureCredential: 'must-not-leak', 'slack.futureCredential': 'must-not-leak' } }, + ); + const metadata = await scope(() => db.getAgentChannelMetadata('channel')); + expect(Object.keys(metadata!).sort()).toEqual([ + 'agentId', + 'channelId', + 'createdBy', + 'credentialMode', + 'revision', + 'slack', + 'status', + 'tenantId', + 'type', + ]); + expect(metadata?.slack).toEqual({ teamId: 'T1', appId: 'A1', botUserId: 'UBOT' }); + expect(JSON.stringify(metadata)).not.toContain('must-not-leak'); +}); + +it('defines bounded cleanup configuration without exposing the algorithmic claim batch as a policy knob', () => { + const config = channelConfigSchema.parse({}); + expect(config.cleanup).toEqual({ batchSize: 25, intervalMs: 1000, maxIdleIntervalMs: 30_000 }); + expect(config.claimBatchSize).toBe(25); + for (const cleanup of [ + { batchSize: 0 }, + { batchSize: 101 }, + { intervalMs: 999 }, + { maxIdleIntervalMs: 300_001 }, + ]) { + expect(channelConfigSchema.safeParse({ cleanup }).success).toBe(false); + } +}); + +it('keeps tenantless work authorized and deletes its closed parent in the application default tenant', async () => { + const defaultScope = (work: () => Promise) => + tenantStorage.run({ tenantId: undefined }, work); + await defaultScope(async () => { + await db.createAgentChannel({ + channelId: 'default-channel', + agentId: 'agent', + createdBy: 'user-a', + type: 'slack', + credentialMode: 'per_user', + status: 'active', + revision: 'v1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' }, + }); + await mongoose.models.ChannelIdentity.create({ + tenantId: '_', + linkId: 'link', + revision: 'link-v1', + state: 'active', + userId: 'user-a', + ticketHash: 'default-ticket', + ticketExpiresAtMs: clock + 60000, + expiresAtMs: clock + 600000, + identity: { + tenantId: '_', + channelId: 'default-channel', + channelRevision: 'v1', + teamId: 'T1', + externalUserId: 'U1', + agentId: 'agent', + }, + }); + await storage.admit( + { + tenantId: '_', + channelId: 'default-channel', + agentId: 'agent', + principal: { + userId: 'user-a', + channelRevision: 'v1', + linkId: 'link', + linkRevision: 'link-v1', + }, + admission: { + eventId: 'default-event', + externalChannelId: 'C1', + externalThreadKey: 'C1:1', + externalMessageKey: 'C1:1', + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + input: 'default tenant input', + }, + }, + clock, + ); + }); + await runtime.initialize(); + expect(retired).toHaveLength(0); + expect(deps.onError).not.toHaveBeenCalled(); + await defaultScope(() => db.beginAgentChannelDeletion('default-channel', 'user-a')); + await restart(); + expect(retired).toHaveLength(1); + expect(retired[0].tenantId).toBe('_'); + expect(await defaultScope(() => db.getAgentChannelMetadata('default-channel'))).toBeNull(); + expect(deps.onError).not.toHaveBeenCalled(); +}); + +it('caps maintenance pages by the storage policy and never turns a credential lookup outage into retirement', async () => { + await admit('one'); + await admit('two'); + deps.getConfig = async () => + channelConfigSchema.parse({ + enabled: true, + maxMetadataPageSize: 1, + cleanup: { batchSize: 2, intervalMs: 60000, maxIdleIntervalMs: 60000 }, + }); + const read = jest.spyOn(storage, 'listPendingMetadata'); + deps.isCredentialCurrent = async () => { + throw new Error('credential store unavailable'); + }; + await runtime.initialize(); + expect(read).toHaveBeenCalledWith(undefined, 1); + expect(retired).toHaveLength(0); + expect(deps.onError).toHaveBeenCalledTimes(1); +}); + +it('holds an invalid issued credential lane until exact retirement confirms provider drain', async () => { + const { run } = await admit('expired-credential'); + const claimed = (await scope(() => storage.claim(tenantId, 'channel', 'gateway', clock, 60000)))!; + const fence = { runId: run.runId, owner: 'gateway', epoch: claimed.leaseEpoch }; + await scope(() => storage.prepareExecution(tenantId, fence, clock)); + await scope(() => + storage.attachResponseCredential( + tenantId, + fence, + { apiKeyId: 'expired-key', encryptedApiKey: 'encrypted-original' }, + clock, + ), + ); + deps.isCredentialCurrent = async (current) => { + expect(current.execution?.apiKeyId).toBe('expired-key'); + return false; + }; + stoppedResult = 'pending'; + await runtime.initialize(); + expect(retired).toHaveLength(1); + expect(retired[0].execution?.apiKeyId).toBe('expired-key'); + expect( + await scope(() => mongoose.models.ChannelLane.findOne({ activeRunId: run.runId }).lean()), + ).not.toBeNull(); + expect((await scope(() => storage.read(tenantId, 'channel', run.runId)))?.state).not.toBe( + 'cancelled', + ); + clock += 60001; + stoppedResult = 'stopped'; + await restart(); + expect(retired).toHaveLength(2); + expect((await scope(() => storage.read(tenantId, 'channel', run.runId)))?.state).toBe( + 'cancelled', + ); + expect( + await scope(() => mongoose.models.ChannelLane.findOne({ activeRunId: run.runId }).lean()), + ).toBeNull(); + expect(deps.onError).not.toHaveBeenCalled(); +}); + +it.each(['read', 'parse'] as const)( + 'retries failed %s initialization with one shared attempt', + async (failure) => { + await admit(); + allowed = false; + const original = deps.getConfig; + let fail = true; + const read = jest.fn(async () => { + if (fail) { + if (failure === 'read') throw new Error('configuration unavailable'); + const invalid = channelConfigSchema.parse({}); + invalid.cleanup.intervalMs = 0; + return invalid; + } + return original(); + }); + deps.getConfig = read; + const first = runtime.initialize(); + expect(runtime.initialize()).toBe(first); + await expect(first).rejects.toThrow(); + fail = false; + const retry = runtime.initialize(); + expect(runtime.initialize()).toBe(retry); + await retry; + expect(retired).toHaveLength(1); + const calls = read.mock.calls.length; + await runtime.initialize(); + expect(read).toHaveBeenCalledTimes(calls); + }, +); + +it('keeps stop permanent after an initialization failure', async () => { + const read = jest.fn(async () => { + throw new Error('configuration unavailable'); + }); + deps.getConfig = read; + await expect(runtime.initialize()).rejects.toThrow('configuration unavailable'); + await runtime.stop(); + await runtime.initialize(); + runtime.wake(); + expect(read).toHaveBeenCalledTimes(1); + expect(retired).toHaveLength(0); +}); + +it('disposes a partially started worker before retrying initialization', async () => { + await admit(); + allowed = false; + const create = recovery.createIdleRecoveryLoop; + const partial = create({ + intervalMs: 60000, + maxIdleIntervalMs: 60000, + scan: async () => true, + onError: jest.fn(), + }); + const start = jest + .spyOn(partial, 'start') + .mockRejectedValueOnce(new Error('initial scan failed')); + const stop = jest.spyOn(partial, 'stop'); + const factory = jest.spyOn(recovery, 'createIdleRecoveryLoop').mockReturnValueOnce(partial); + try { + await expect(runtime.initialize()).rejects.toThrow('initial scan failed'); + expect(start).toHaveBeenCalledTimes(1); + expect(stop).toHaveBeenCalledTimes(1); + await runtime.initialize(); + expect(retired).toHaveLength(1); + expect(factory).toHaveBeenCalledTimes(2); + } finally { + factory.mockRestore(); + start.mockRestore(); + stop.mockRestore(); + } +}); + +it('does not reopen when a pending initialization rejects during shutdown', async () => { + let reject!: (error: Error) => void; + const gate = new Promise>>((_resolve, fail) => { + reject = fail; + }); + const read = jest.fn(() => gate); + deps.getConfig = read; + const initializing = runtime.initialize(); + const stopping = runtime.stop(); + reject(new Error('startup unavailable')); + await expect(initializing).rejects.toThrow('startup unavailable'); + await stopping; + await runtime.initialize(); + expect(read).toHaveBeenCalledTimes(1); + expect(retired).toHaveLength(0); +}); + +it.each([false, true])( + 'releases an expired Response lane only after canonical retirement proof: pending=%s', + async (pending) => { + await admit(); + const claimed = (await scope(() => + storage.claim(tenantId, 'channel', 'worker', clock, 60000), + ))!; + const fence = { runId: claimed.runId, owner: claimed.leaseOwner!, epoch: claimed.leaseEpoch }; + const prepared = await scope(() => storage.prepareExecution(tenantId, fence, clock)); + await scope(() => + storage.attachResponseCredential( + tenantId, + fence, + { apiKeyId: 'issued', encryptedApiKey: 'encrypted' }, + clock, + ), + ); + const responses = createResponseStorage(mongoose); + const execution = prepared.execution!; + const owner = { userId: prepared.principal.userId, tenantId }; + const responseId = `resp_${prepared.runId}`; + const reservation = await scope(() => + responses.reserve({ + ...owner, + responseId, + requestKey: `channel:${prepared.runId}`, + sourceKeyId: 'issued', + fingerprint: 'cleanup', + agentId: prepared.agentId, + requestJSON: JSON.stringify({ + model: prepared.agentId, + input: prepared.admission.input, + background: true, + store: true, + ...(execution.parentMessageId === '00000000-0000-0000-0000-000000000000' + ? {} + : { previous_response_id: execution.parentMessageId }), + }), + conversationId: execution.conversationId, + parentMessageId: execution.parentMessageId, + }), + ); + if (!reservation.ok) throw new Error('Reservation failed'); + await scope(() => storage.bindResponse(tenantId, fence, responseId, clock)); + await scope(() => + mongoose.models.StoredResponse.updateOne( + { responseId }, + { $set: { expiredAt: new Date(0), ...(pending ? { generationCreatedAt: 1234 } : {}) } }, + ), + ); + const jobs = { + getJob: jest.fn(async () => undefined), + abortJob: jest.fn(), + finishProviderTerminalPersistence: jest.fn(), + }; + const adapter = createChannelExecutionAdapter({ + responseProjection: async () => { + throw new Error('Cleanup must not project or execute'); + }, + responseRetirement: createChannelResponseRetirement({ + storage: responses, + recovery: storage, + retireResponse: createResponseRetirement({ storage: responses, jobs }), + }), + }); + deps.retireForAuthorityLoss = adapter.retireForAuthorityLoss; + deps.isCredentialCurrent = async () => false; + await runtime.initialize(); + await runtime.stop(); + const next = await admit('next'); + const nextClaim = await scope(() => + storage.claim(tenantId, 'channel', 'next-worker', clock + 60001, 60000), + ); + if (pending) { + expect(nextClaim?.runId).toBe(claimed.runId); + expect(nextClaim?.errorCode).toBe('channel_authority_changed'); + } else expect(nextClaim?.runId).toBe(next.run.runId); + expect(jobs.abortJob).not.toHaveBeenCalled(); + }, +); diff --git a/packages/api/src/channels/cleanup.ts b/packages/api/src/channels/cleanup.ts new file mode 100644 index 0000000000..89dbcb426d --- /dev/null +++ b/packages/api/src/channels/cleanup.ts @@ -0,0 +1,155 @@ +import { randomUUID } from 'node:crypto'; +import { channelConfigSchema } from 'librechat-data-provider'; +import { runAsSystem, tenantStorage } from '@librechat/data-schemas'; +import type { AgentChannelMethods, ChannelRunStorage } from '@librechat/data-schemas'; +import type { ChannelConfig } from 'librechat-data-provider'; +import type { ChannelRunDependencies } from './runs'; +import { createIdleRecoveryLoop } from '~/agents/recovery'; +import { hasChannelRunAuthority } from './runs'; + +export interface ChannelCleanupDependencies { + storage: ChannelRunStorage; + channels: Pick< + AgentChannelMethods, + 'getAgentChannelMetadata' | 'listDeletingAgentChannels' | 'deleteAgentChannel' + >; + resolvePrincipal: ChannelRunDependencies['resolvePrincipal']; + retireForAuthorityLoss: ChannelRunDependencies['retireForAuthorityLoss']; + isCredentialCurrent: ( + run: Parameters[0], + ) => Promise; + deleteApiKey: (id: string, userId: string) => Promise; + getConfig: () => Promise; + onError: (error: unknown) => void; + now?: () => number; +} + +/** Private recovery survives disabled features and missing gateway/channel rows. + * Only metadata crosses system context; cleanup cannot enqueue or deliver. */ +export function createChannelCleanupRuntime(deps: ChannelCleanupDependencies) { + const owner = `channel-cleanup-${randomUUID()}`; + const now = deps.now ?? Date.now; + let worker: ReturnType | undefined; + let initialization: Promise | undefined; + let stopped = false; + let runCursor: string | undefined; + let channelCursor: string | undefined; + return { + initialize(): Promise { + if (stopped) return Promise.resolve(); + if (initialization) return initialization; + initialization = (async () => { + const initial = channelConfigSchema.parse((await deps.getConfig()) ?? {}); + if (stopped) return; + worker = createIdleRecoveryLoop({ + ...initial.cleanup, + onError: deps.onError, + scan: async () => { + if (stopped) return true; + const config = channelConfigSchema.parse((await deps.getConfig()) ?? {}); + const pageSize = Math.min(config.cleanup.batchSize, config.maxMetadataPageSize); + const [runs, deleting] = await runAsSystem(() => + Promise.all([ + deps.storage.listPendingMetadata(runCursor, pageSize), + deps.channels.listDeletingAgentChannels(channelCursor, pageSize), + ]), + ); + runCursor = runs.next; + channelCursor = deleting.next; + let idle = !runs.next && !deleting.next; + for (const target of runs.runs) { + if (stopped) break; + await tenantStorage.run( + { tenantId: target.tenantId === '_' ? undefined : target.tenantId }, + async () => { + try { + const run = await deps.storage.read( + target.tenantId, + target.channelId, + target.runId, + ); + if (!run) return; + const channel = await deps.channels.getAgentChannelMetadata(target.channelId); + if ( + config.enabled && + channel && + (await hasChannelRunAuthority(channel, run, deps.resolvePrincipal)) && + (await deps.isCredentialCurrent(run)) + ) + return; + const claimed = await deps.storage.claimAuthorityLost( + run, + owner, + now(), + config.leaseMs, + ); + if (!claimed) return; + idle = false; + if ((await deps.retireForAuthorityLoss(claimed)) !== 'stopped') return; + await deps.storage.finishAuthorityLost( + claimed.tenantId, + { + runId: claimed.runId, + owner, + epoch: claimed.leaseEpoch, + }, + now(), + ); + } catch (error) { + idle = false; + deps.onError(error); + } + }, + ); + } + for (const target of deleting.channels) { + if (stopped) break; + await tenantStorage.run({ tenantId: target.tenantId || undefined }, async () => { + try { + const channel = await deps.channels.getAgentChannelMetadata(target.channelId); + if (channel?.status !== 'deleting') return; + if (channel.apiKeyId) + await deps.deleteApiKey(channel.apiKeyId, channel.createdBy); + const result = await deps.channels.deleteAgentChannel( + channel.channelId, + channel.createdBy, + ); + if (result.status !== 'not_found') idle = false; + } catch (error) { + idle = false; + deps.onError(error); + } + }); + } + return idle; + }, + }); + await worker.start(); + if (stopped) await worker.stop(); + })().catch(async (error: unknown) => { + const failed = worker; + worker = undefined; + try { + await failed?.stop(); + } catch { + /* Preserve the initialization failure after attempting owned worker disposal. */ + } + initialization = undefined; + throw error; + }); + return initialization; + }, + wake(): void { + worker?.wake(); + }, + async stop(): Promise { + stopped = true; + try { + await initialization; + } catch { + /* Failed initialization has no running worker. */ + } + await worker?.stop(); + }, + }; +} diff --git a/packages/api/src/channels/credentials.ts b/packages/api/src/channels/credentials.ts new file mode 100644 index 0000000000..cfb1ba0a93 --- /dev/null +++ b/packages/api/src/channels/credentials.ts @@ -0,0 +1,209 @@ +import { z } from 'zod'; +import { Constants } from 'librechat-data-provider'; +import type { + AgentApiKeyBinding, + AgentApiKeyListItem, + AgentChannelRecord, + ChannelRunRecord, + ChannelRunStorage, + ResponseOwner, + ResponseStorage, + IUser, +} from '@librechat/data-schemas'; +import type { Request } from 'express'; +import type { ChannelRunDependencies } from './runs'; +import { hasChannelRunAuthority } from './runs'; + +export interface ChannelResponseCredentialDependencies { + encrypt: (value: string) => Promise; + decrypt: (value: string) => Promise; + storage: Pick; + responses: Pick; + getChannel: (id: string) => Promise; + resolvePrincipal: ChannelRunDependencies['resolvePrincipal']; + createKey: (input: { + userId: string; + name: string; + expiresAt: Date; + binding: AgentApiKeyBinding; + }) => Promise<{ id: string; key: string }>; + getKey: (keyId: string, userId: string) => Promise; + deleteKey: (keyId: string, userId: string) => Promise; + lifetimeMs: () => Promise; + now?: () => number; +} + +const requestSchema = z + .object({ + model: z.string(), + input: z.string(), + background: z.literal(true), + store: z.literal(true), + stream: z.literal(false).optional(), + previous_response_id: z.string().optional(), + }) + .strict(); +const bindingFor = (run: ChannelRunRecord): AgentApiKeyBinding => ({ + kind: 'channel_run', + id: `${run.channelId}/${run.runId}`, +}); +const previousResponse = (run: ChannelRunRecord) => + run.execution?.parentMessageId === Constants.NO_PARENT + ? undefined + : run.execution?.parentMessageId; + +/** Raw keys cross only the guarded gateway boundary; durable state retains encrypted keys and immutable grant references. */ +export function createChannelResponseCredentials(deps: ChannelResponseCredentialDependencies) { + const now = deps.now ?? Date.now; + async function current( + binding: AgentApiKeyBinding, + keyId: string, + owner: ResponseOwner, + agentId?: string, + ) { + if (binding.kind !== 'channel_run' || !keyId) return null; + const [channelId, runId, extra] = binding.id.split('/'); + if (!channelId || !runId || extra != null) return null; + const run = await deps.storage.read(owner.tenantId || '_', channelId, runId); + if ( + !run || + run.principal.userId !== owner.userId || + run.execution?.apiKeyId !== keyId || + run.execution.protocol !== 'responses' || + (agentId != null && run.agentId !== agentId) + ) + return null; + const [authorized, key] = await Promise.all([ + deps + .getChannel(channelId) + .then((channel) => + channel ? hasChannelRunAuthority(channel, run, deps.resolvePrincipal) : false, + ), + deps.getKey(keyId, run.principal.userId), + ]); + // A key read early must still be unexpired after the independent policy lookup finishes. + return authorized && eligibleKey(run, key) ? run : null; + } + function eligibleKey(run: ChannelRunRecord, key: AgentApiKeyListItem | null): boolean { + const execution = run.execution; + if (!execution?.apiKeyId || execution.protocol !== 'responses' || !execution.encryptedApiKey) + return false; + const binding = bindingFor(run); + return Boolean( + key && + key.id === execution.apiKeyId && + key.binding?.kind === binding.kind && + key.binding.id === binding.id && + (key.expiresAt == null || + (Number.isFinite(new Date(key.expiresAt).getTime()) && + new Date(key.expiresAt).getTime() > now())), + ); + } + async function isCurrent(run: ChannelRunRecord): Promise { + const execution = run.execution; + if (!execution?.apiKeyId) return true; + if (execution.protocol !== 'responses' || !execution.encryptedApiKey) return false; + return eligibleKey(run, await deps.getKey(execution.apiKeyId, run.principal.userId)); + } + async function reveal(run: ChannelRunRecord): Promise { + const execution = run.execution; + if (!execution?.apiKeyId || !execution.encryptedApiKey) + throw new Error('Response credential unavailable'); + if (!(await isCurrent(run))) throw new Error('Response credential authority changed'); + return deps.decrypt(execution.encryptedApiKey); + } + return { + isCurrent: (run: ChannelRunRecord): Promise => isCurrent(run), + async provision(run: ChannelRunRecord): Promise { + if (run.execution?.apiKeyId) return reveal(run); + if ( + !run.execution || + run.execution.deliveryKey || + run.execution.enqueueAttemptedAt || + run.cancellationRequested || + !run.leaseOwner + ) + throw new Error('Response credential admission changed'); + const lifetimeMs = await deps.lifetimeMs(); + if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs <= 0) + throw new Error('Invalid response credential policy'); + const key = await deps.createKey({ + userId: run.principal.userId, + name: 'Channel response run', + expiresAt: new Date(now() + lifetimeMs), + binding: bindingFor(run), + }); + let saved: ChannelRunRecord; + try { + saved = await deps.storage.attachResponseCredential( + run.tenantId, + { runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch }, + { apiKeyId: key.id, encryptedApiKey: await deps.encrypt(key.key) }, + now(), + ); + } catch (error) { + const currentRun = await deps.storage.read(run.tenantId, run.channelId, run.runId); + const winner = currentRun?.execution; + const sameAdmission = + currentRun && + winner?.apiKeyId && + winner.encryptedApiKey && + winner.protocol === 'responses' && + currentRun.principal.userId === run.principal.userId && + currentRun.agentId === run.agentId && + winner.requestId === run.execution.requestId && + currentRun.leaseOwner === run.leaseOwner && + currentRun.leaseEpoch === run.leaseEpoch; + if (!sameAdmission) { + if (winner?.apiKeyId !== key.id) await deps.deleteKey(key.id, run.principal.userId); + throw error; + } + saved = currentRun; + } + if (saved.execution?.apiKeyId !== key.id) await deps.deleteKey(key.id, run.principal.userId); + return reveal(saved); + }, + async authorizeSource( + binding: AgentApiKeyBinding, + keyId: string, + owner: ResponseOwner, + agentId: string, + ): Promise { + return Boolean(await current(binding, keyId, owner, agentId)); + }, + async authorizeRequest(input: { + keyId: string; + binding: AgentApiKeyBinding; + user: IUser; + request: Request; + }): Promise { + const owner = { userId: String(input.user._id), tenantId: input.user.tenantId }; + const run = await current(input.binding, input.keyId, owner); + if (!run) return false; + const pathname = input.request.originalUrl.split('?')[0]; + if (input.request.method === 'POST' && pathname === '/api/agents/v1/responses') { + if (run.cancellationRequested) return false; + const parsed = requestSchema.safeParse(input.request.body); + return ( + parsed.success && + parsed.data.model === run.agentId && + parsed.data.input === run.admission.input && + parsed.data.previous_response_id === previousResponse(run) && + input.request.headers['idempotency-key'] === `channel:${run.runId}` + ); + } + const path = /^\/api\/agents\/v1\/responses\/(resp_[\w-]+)(\/cancel)?$/.exec(pathname); + if (!path || (path[2] ? input.request.method !== 'POST' : input.request.method !== 'GET')) + return false; + if (path[2] && !run.cancellationRequested) return false; + const response = await deps.responses.read(owner, path[1]); + return Boolean( + response && + response.agentId === run.agentId && + response.requestKey === `channel:${run.runId}` && + response.sourceKeyId === input.keyId && + (run.execution?.responseId == null || run.execution.responseId === response.responseId), + ); + }, + }; +} diff --git a/packages/api/src/channels/errors.spec.ts b/packages/api/src/channels/errors.spec.ts new file mode 100644 index 0000000000..5ef5a7ba18 --- /dev/null +++ b/packages/api/src/channels/errors.spec.ts @@ -0,0 +1,19 @@ +import { ChannelExecutionError as CompatibilityError } from './execution'; +import { ChannelExecutionError } from './errors'; + +it.each([ + 'execution_not_prepared', + 'execution_identity_changed', + 'dispatch_outcome_unknown', + 'generation_missing', + 'generation_replaced', + 'stop_unconfirmed', +] as const)('preserves error identity and diagnostics for %s', (code) => { + expect(CompatibilityError).toBe(ChannelExecutionError); + const error = new ChannelExecutionError(code); + expect(error).toBeInstanceOf(Error); + expect(error).toBeInstanceOf(CompatibilityError); + expect(error.name).toBe('ChannelExecutionError'); + expect(error.message).toBe(code); + expect(error.code).toBe(code); +}); diff --git a/packages/api/src/channels/errors.ts b/packages/api/src/channels/errors.ts new file mode 100644 index 0000000000..c16639ed12 --- /dev/null +++ b/packages/api/src/channels/errors.ts @@ -0,0 +1,14 @@ +export class ChannelExecutionError extends Error { + constructor( + readonly code: + | 'execution_not_prepared' + | 'execution_identity_changed' + | 'dispatch_outcome_unknown' + | 'generation_missing' + | 'generation_replaced' + | 'stop_unconfirmed', + ) { + super(code); + this.name = 'ChannelExecutionError'; + } +} diff --git a/packages/api/src/channels/execution.spec.ts b/packages/api/src/channels/execution.spec.ts new file mode 100644 index 0000000000..98a2112ee3 --- /dev/null +++ b/packages/api/src/channels/execution.spec.ts @@ -0,0 +1,256 @@ +import express from 'express'; +import request from 'supertest'; +import mongoose from 'mongoose'; +import { Constants } from 'librechat-data-provider'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createChannelRunStorage, + tenantStorage, +} from '@librechat/data-schemas'; +import type { ChannelRunRecord } from '@librechat/data-schemas'; +import type { ChannelExecutionDependencies } from './execution'; +import { createChannelExecutionAdapter } from './execution'; +import { createChannelHandlers } from './service'; +import { createChannelRunHandlers } from './runs'; + +const scope = (work: () => Promise) => tenantStorage.run({ tenantId: 'tenant-a' }, work); +const makeRun = (): ChannelRunRecord => ({ + tenantId: 'tenant-a', + channelId: 'channel-a', + agentId: 'agent-a', + runId: 'channel-run-a', + laneId: 'lane-a', + state: 'executing', + sequence: 1, + receivedAt: Date.now(), + payloadHash: 'hash-a', + principal: { + userId: 'user-a', + channelRevision: 'revision-a', + linkId: 'link-a', + linkRevision: 'link-rev-a', + }, + admission: { + eventId: 'event-a', + externalChannelId: 'C1', + externalThreadKey: 'C1:1.001', + externalMessageKey: 'C1:1.001', + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + input: 'hello', + }, + cancellationRequested: false, + receipts: [], + leaseEpoch: 1, + leaseExpiresAt: Date.now() + 60_000, + execution: { + requestId: 'channel-run-a', + conversationId: 'conversation-a', + parentMessageId: String(Constants.NO_PARENT), + }, +}); + +describe('mounted service-token advance cannot execute an unbound run', () => { + let mongo: MongoMemoryReplSet; + beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await mongoose.connect(mongo.getUri()); + createModels(mongoose); + }); + afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); + }); + it('rejects legacy and unprepared state behind the actual service guard without starting work', () => + scope(async () => { + const fixture = makeRun(); + await createMethods(mongoose).createAgentChannel({ + channelId: fixture.channelId, + tenantId: fixture.tenantId, + agentId: fixture.agentId, + createdBy: fixture.principal.userId, + type: 'slack', + status: 'active', + credentialMode: 'per_user', + revision: fixture.principal.channelRevision, + slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' }, + }); + await mongoose.models.ChannelIdentity.create({ + tenantId: fixture.tenantId, + linkId: fixture.principal.linkId, + revision: fixture.principal.linkRevision, + userId: fixture.principal.userId, + state: 'active', + ticketHash: 'ticket', + ticketExpiresAtMs: Date.now() + 60000, + expiresAtMs: Date.now() + 60000, + identity: { + tenantId: fixture.tenantId, + channelId: fixture.channelId, + channelRevision: fixture.principal.channelRevision, + teamId: 'T1', + externalUserId: 'U1', + agentId: fixture.agentId, + }, + }); + const storage = createChannelRunStorage(mongoose); + await storage.admit( + { + tenantId: fixture.tenantId, + channelId: fixture.channelId, + agentId: fixture.agentId, + principal: fixture.principal, + admission: fixture.admission, + }, + Date.now(), + ); + const run = (await storage.claim( + fixture.tenantId, + fixture.channelId, + 'worker', + Date.now(), + 60_000, + ))!; + const token = 'test-service-credential-at-least-32-characters'; + const handlers = createChannelHandlers({ + db: createMethods(mongoose), + serviceToken: () => token, + config: async () => ({ enabled: true }), + getAgent: async () => ({ id: fixture.agentId }), + deleteApiKey: async () => true, + verifySlackToken: async () => null, + logError: jest.fn(), + }); + const responseProjection = jest.fn(); + const adapter = createChannelExecutionAdapter({ + responseRetirement: jest.fn(), + responseProjection, + }); + const runs = createChannelRunHandlers({ + storage, + guard: handlers.service, + resolvePrincipal: async () => fixture.principal, + advance: adapter.advance, + retireForAuthorityLoss: adapter.retireForAuthorityLoss, + }); + const app = express(); + app.use(express.json()); + app.post('/service/:channelId/runs/:runId/advance', runs.advance); + const route = `/service/${run.channelId}/runs/${run.runId}/advance`; + const body = { fence: { runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch } }; + await request(app).post(route).send(body).expect(401); + for (let attempt = 0; attempt < 2; attempt++) { + await request(app) + .post(route) + .auth(token, { type: 'bearer' }) + .set('X-Tenant-Id', fixture.tenantId) + .send(body) + .expect(409, { error: 'execution_not_prepared' }); + } + const held = await storage.read(run.tenantId, run.channelId, run.runId); + expect(held?.execution?.enqueueAttemptedAt).toBeUndefined(); + expect(held?.execution?.deliveryKey).toBeUndefined(); + expect(held?.snapshot).toBeUndefined(); + expect(responseProjection).not.toHaveBeenCalled(); + })); +}); + +describe('Responses-only Channels execution', () => { + let run: ChannelRunRecord; + let deps: ChannelExecutionDependencies; + let adapter: ReturnType; + + beforeEach(() => { + run = makeRun(); + deps = { + responseProjection: jest.fn(async () => ({ + snapshot: { status: 'running' as const, text: '', elapsedMs: 1 }, + })), + responseRetirement: jest.fn(async () => 'pending' as const), + }; + adapter = createChannelExecutionAdapter(deps); + }); + + function loseAuthority() { + run.cancellationRequested = true; + run.errorCode = 'channel_authority_changed'; + } + + it.each(['unprepared', 'unbound', 'legacy_dispatched', 'responses_unbound'] as const)( + 'refuses %s advance without projecting a Response', + async (kind) => { + if (kind === 'unprepared') run.execution = undefined; + if (kind === 'legacy_dispatched') run.execution!.deliveryKey = 'old-trigger'; + if (kind === 'responses_unbound') run.execution!.protocol = 'responses'; + await expect(adapter.advance(run)).rejects.toMatchObject({ code: 'execution_not_prepared' }); + expect(deps.responseProjection).not.toHaveBeenCalled(); + }, + ); + + it('delegates a bound Response to its canonical projection and retirement', async () => { + run.execution = { ...run.execution!, protocol: 'responses', responseId: 'resp_saved' }; + await expect(adapter.advance(run)).resolves.toMatchObject({ snapshot: { status: 'running' } }); + expect(deps.responseProjection).toHaveBeenCalledWith(run); + loseAuthority(); + await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('pending'); + expect(deps.responseRetirement).toHaveBeenCalledWith(run); + }); + + it.each(['not_cancelled', 'different_reason'] as const)( + 'rejects %s cleanup before consulting Responses', + async (kind) => { + loseAuthority(); + if (kind === 'not_cancelled') run.cancellationRequested = false; + if (kind === 'different_reason') run.errorCode = 'other'; + await expect(adapter.retireForAuthorityLoss(run)).rejects.toMatchObject({ + code: 'execution_identity_changed', + }); + expect(deps.responseRetirement).not.toHaveBeenCalled(); + }, + ); + + it.each(['unprepared', 'unprovisioned', 'responses_unprovisioned'] as const)( + 'retires %s authority loss without a possible provider admission', + async (kind) => { + loseAuthority(); + if (kind === 'unprepared') run.execution = undefined; + if (kind === 'responses_unprovisioned') run.execution!.protocol = 'responses'; + await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('stopped'); + expect(deps.responseRetirement).not.toHaveBeenCalled(); + }, + ); + + it('reconciles an issued credential even when Response binding has not finished', async () => { + loseAuthority(); + run.execution = { ...run.execution!, protocol: 'responses', apiKeyId: 'issued' }; + await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('pending'); + expect(deps.responseRetirement).toHaveBeenCalledWith(run); + }); + + it.each([ + 'deliveryKey', + 'enqueueAttemptedAt', + 'generationCreatedAt', + 'apiKeyId', + 'responseId', + ] as const)( + 'holds unknown execution with %s instead of guessing that it stopped', + async (field) => { + loseAuthority(); + Object.assign(run.execution!, { [field]: field.endsWith('At') ? 100 : 'unknown' }); + await expect(adapter.retireForAuthorityLoss(run)).rejects.toMatchObject({ + code: 'execution_identity_changed', + }); + expect(deps.responseRetirement).not.toHaveBeenCalled(); + }, + ); + + it('preserves canonical retirement errors for recovery to retry', async () => { + loseAuthority(); + run.execution = { ...run.execution!, protocol: 'responses', responseId: 'resp_saved' }; + const error = new Error('storage_unavailable'); + jest.mocked(deps.responseRetirement).mockRejectedValue(error); + await expect(adapter.retireForAuthorityLoss(run)).rejects.toBe(error); + }); +}); diff --git a/packages/api/src/channels/execution.ts b/packages/api/src/channels/execution.ts new file mode 100644 index 0000000000..380bc0fcc2 --- /dev/null +++ b/packages/api/src/channels/execution.ts @@ -0,0 +1,43 @@ +import type { ChannelRunSnapshot } from 'librechat-data-provider'; +import type { ChannelRunRecord } from '@librechat/data-schemas'; +import type { createChannelResponseProjection, createChannelResponseRetirement } from './responses'; +import { ChannelExecutionError } from './errors'; + +export { ChannelExecutionError } from './errors'; + +export interface ChannelExecutionDependencies { + responseProjection: ReturnType; + responseRetirement: ReturnType; +} + +/** Execution and cancellation belong to Responses; Channels only projects delivery state. */ +export function createChannelExecutionAdapter(deps: ChannelExecutionDependencies) { + return { + async retireForAuthorityLoss(run: ChannelRunRecord): Promise<'stopped' | 'pending'> { + if (!run.cancellationRequested || run.errorCode !== 'channel_authority_changed') + throw new ChannelExecutionError('execution_identity_changed'); + if (!run.execution) return 'stopped'; + const execution = run.execution; + // A credential permits Response admission even before its ID has been bound. + // Only an unprovisioned run can be retired without consulting Responses. + if ( + !execution.apiKeyId && + !execution.responseId && + execution.generationCreatedAt == null && + execution.deliveryKey == null && + execution.enqueueAttemptedAt == null + ) + return 'stopped'; + if (execution.protocol !== 'responses') + throw new ChannelExecutionError('execution_identity_changed'); + return deps.responseRetirement(run); + }, + async advance( + run: ChannelRunRecord, + ): Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }> { + if (run.execution?.protocol !== 'responses' || !run.execution.responseId) + throw new ChannelExecutionError('execution_not_prepared'); + return deps.responseProjection(run); + }, + }; +} diff --git a/packages/api/src/channels/responses.spec.ts b/packages/api/src/channels/responses.spec.ts new file mode 100644 index 0000000000..d3877aed2f --- /dev/null +++ b/packages/api/src/channels/responses.spec.ts @@ -0,0 +1,389 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createResponseStorage, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { + ChannelRunRecord, + ResponseStorage, + StoredResponseRecord, +} from '@librechat/data-schemas'; +import type { Response } from '~/agents/responses/types'; +import { + createResponseCompletion, + createResponseExecutionConfirmation, + createResponseExecutionRelease, +} from '~/agents/responses/completion'; +import { + buildPendingBackgroundResponse, + createBackgroundResponseService, +} from '~/agents/responses/background'; +import { createChannelResponseProjection, createChannelResponseRetirement } from './responses'; +import { createResponseRetirement } from '~/agents/responses/retirement'; +import { GenerationJobManager } from '~/stream'; + +let mongo: MongoMemoryReplSet; +let storage: ResponseStorage; +let stored: StoredResponseRecord; +let run: ChannelRunRecord; +const owner = { userId: 'user', tenantId: 'projection-test' }; +const scoped = (work: () => Promise) => tenantStorage.run(owner, work); +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ + replSet: { count: 1 }, + instanceOpts: [{ args: ['--nounixsocket'] }], + }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + storage = createResponseStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + await mongoose.models.Conversation.deleteMany({}); + await mongoose.models.ResponseScope?.deleteMany({}); + }); + const reserved = await scoped(() => + storage.reserve({ + ...owner, + responseId: 'resp_saved', + requestKey: 'channel:run', + fingerprint: 'request', + agentId: 'agent', + sourceKeyId: 'key', + requestJSON: JSON.stringify({ + model: 'agent', + input: 'hello', + background: true, + store: true, + }), + conversationId: 'conversation', + parentMessageId: 'root', + }), + ); + if (!reserved.ok) throw new Error('Reservation failed'); + stored = reserved.record; + run = { + tenantId: owner.tenantId, + channelId: 'channel', + agentId: 'agent', + runId: 'run', + laneId: 'lane', + state: 'executing', + sequence: 1, + payloadHash: 'payload', + receivedAt: 1000, + principal: { userId: owner.userId, channelRevision: 'revision' }, + admission: { + eventId: 'event', + externalChannelId: 'C1', + externalThreadKey: 'C1:1.0', + externalMessageKey: 'C1:1.1', + actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' }, + input: 'hello', + }, + cancellationRequested: false, + receipts: [], + leaseEpoch: 1, + leaseExpiresAt: Date.now() + 10000, + execution: { + protocol: 'responses', + responseId: stored.responseId, + requestId: 'run', + conversationId: stored.conversationId, + parentMessageId: stored.parentMessageId, + }, + }; +}); +function fixture() { + let authorized = true; + const cancel = jest.fn(async () => {}); + const service = createBackgroundResponseService({ + storage, + recoverTerminal: createResponseCompletion({ + storage, + cleanupCheckpoint: async () => {}, + confirmExecution: createResponseExecutionConfirmation(GenerationJobManager), + releaseExecution: createResponseExecutionRelease(GenerationJobManager), + }).finish, + authorize: async () => authorized, + enqueue: async () => { + throw new Error('Projection must not enqueue'); + }, + resolveContinuation: async () => { + throw new Error('Projection must not rebuild history'); + }, + observe: async () => null, + cancel, + }); + return { + project: createChannelResponseProjection({ + storage, + readResponse: service.retrieve, + cancelResponse: service.cancel, + now: () => 6000, + }), + cancel, + revoke: () => { + authorized = false; + }, + }; +} +async function publish(response: Response) { + const result = await storage.publish(owner, stored.responseId, stored.revision, { + status: response.status, + responseJSON: JSON.stringify(response), + generationCreatedAt: 1234, + }); + if (!result.ok) throw new Error('Publication failed'); + stored = result.record; +} + +it('carries a nonterminal recovery warning without publishing stale actions, output or enqueueing work', async () => { + await scoped(async () => { + await publish({ + ...buildPendingBackgroundResponse(stored), + status: 'in_progress', + recovery: 'execution_state_unavailable', + }); + const f = fixture(); + const result = await f.project(run); + expect(result.snapshot).toMatchObject({ + status: 'running', + text: '', + recovery: 'execution_state_unavailable', + }); + expect(result.snapshot.pendingAction).toBeUndefined(); + expect(f.cancel).not.toHaveBeenCalled(); + }); +}); + +it('projects only the canonical final assistant text and actual usage', () => + scoped(async () => { + const f = fixture(); + const response: Response = { + ...buildPendingBackgroundResponse(stored), + status: 'completed', + completed_at: 5, + output: [ + { + id: 'message', + type: 'message', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: '# Final answer', annotations: [], logprobs: [] }], + }, + { + id: 'call', + type: 'function_call', + call_id: 'tool', + name: 'private_tool', + arguments: 'PRIVATE arguments', + status: 'completed', + }, + ], + usage: { + input_tokens: 10, + output_tokens: 2, + total_tokens: 12, + input_tokens_details: { cached_tokens: 3 }, + output_tokens_details: { reasoning_tokens: 1 }, + }, + }; + await publish(response); + const result = await f.project(run); + expect(result).toMatchObject({ + generationCreatedAt: 1234, + snapshot: { + responseId: 'resp_saved', + status: 'completed', + text: '# Final answer', + elapsedMs: 4000, + usage: { + inputTokens: 10, + outputTokens: 2, + totalTokens: 12, + cachedTokens: 3, + reasoningTokens: 1, + }, + }, + }); + expect(JSON.stringify(result)).not.toContain('PRIVATE'); + expect(f.cancel).not.toHaveBeenCalled(); + })); + +it('never presents intermediate output as the final Slack answer', () => + scoped(async () => { + const f = fixture(); + await publish({ + ...buildPendingBackgroundResponse(stored), + status: 'in_progress', + output: [ + { + id: 'message', + type: 'message', + role: 'assistant', + status: 'in_progress', + content: [{ type: 'output_text', text: 'Still changing', annotations: [], logprobs: [] }], + }, + ], + }); + expect((await f.project(run)).snapshot).toMatchObject({ status: 'running', text: '' }); + })); + +it('keeps a cancellation request running until the Responses runtime confirms settlement', () => + scoped(async () => { + const f = fixture(); + await publish({ ...buildPendingBackgroundResponse(stored), status: 'in_progress' }); + run.cancellationRequested = true; + expect((await f.project(run)).snapshot.status).toBe('running'); + expect(f.cancel).toHaveBeenCalledTimes(1); + await publish({ + ...buildPendingBackgroundResponse(stored), + status: 'cancelled', + completed_at: 5, + }); + expect((await f.project(run)).snapshot.status).toBe('cancelled'); + })); + +it('honors current source credential authority before returning output', () => + scoped(async () => { + const f = fixture(); + f.revoke(); + await expect(f.project(run)).rejects.toMatchObject({ code: 'execution_identity_changed' }); + })); + +it.each(['responseId', 'conversationId', 'parentMessageId', 'generationCreatedAt'])( + 'rejects changed %s rather than reading the latest conversation output', + (field) => + scoped(async () => { + const f = fixture(); + run.execution = { + ...run.execution!, + [field]: field === 'generationCreatedAt' ? 5678 : 'other', + }; + await expect(f.project(run)).rejects.toMatchObject({ code: 'execution_identity_changed' }); + }), +); + +it.each(['expired', 'closed'] as const)( + 'retires a %s bound Response through cleanup identity without disclosing payload', + (state) => + scoped(async () => { + run.cancellationRequested = true; + run.errorCode = 'channel_authority_changed'; + run.execution!.apiKeyId = 'key'; + if (state === 'expired') + await mongoose.models.StoredResponse.updateOne( + { responseId: stored.responseId }, + { $set: { expiredAt: new Date(0) } }, + ); + else { + await mongoose.models.Conversation.create({ + user: owner.userId, + tenantId: owner.tenantId, + conversationId: stored.conversationId, + endpoint: 'agents', + }); + await expect( + createMethods(mongoose).deleteConvos( + owner.userId, + { conversationId: stored.conversationId }, + { + beforeDelete: async () => { + throw new Error('erasure paused after scope closure'); + }, + }, + ), + ).rejects.toThrow('erasure paused after scope closure'); + } + expect(await storage.read(owner, stored.responseId)).toBeNull(); + const jobs = { + getJob: jest.fn(async () => undefined), + abortJob: jest.fn(), + finishProviderTerminalPersistence: jest.fn(), + }; + const retire = createChannelResponseRetirement({ + storage, + recovery: { recoverResponse: jest.fn() }, + retireResponse: createResponseRetirement({ storage, jobs }), + }); + await expect(retire(run)).resolves.toBe('stopped'); + expect(jobs.abortJob).not.toHaveBeenCalled(); + }), +); + +it.each([ + { requestKey: 'foreign' }, + { sourceKeyId: 'foreign' }, + { parentMessageId: 'foreign' }, + { agentId: 'foreign' }, + { conversationId: 'foreign' }, +])('refuses foreign cleanup binding metadata after expiry: %j', (change) => + scoped(async () => { + run.cancellationRequested = true; + run.errorCode = 'channel_authority_changed'; + run.execution!.apiKeyId = 'key'; + await mongoose.models.StoredResponse.collection.updateOne( + { responseId: stored.responseId }, + { $set: { expiredAt: new Date(0), ...change } }, + ); + const retireResponse = jest.fn(async () => 'stopped' as const); + const retire = createChannelResponseRetirement({ + storage, + recovery: { recoverResponse: jest.fn() }, + retireResponse, + }); + await expect(retire(run)).rejects.toMatchObject({ code: 'execution_identity_changed' }); + expect(retireResponse).not.toHaveBeenCalled(); + }), +); + +it('preserves pending retirement when an expired generation has no drain proof', () => + scoped(async () => { + run.cancellationRequested = true; + run.errorCode = 'channel_authority_changed'; + run.execution!.apiKeyId = 'key'; + await mongoose.models.StoredResponse.updateOne( + { responseId: stored.responseId }, + { $set: { expiredAt: new Date(0), generationCreatedAt: 1234 } }, + ); + const jobs = { + getJob: jest.fn(async () => undefined), + abortJob: jest.fn(), + finishProviderTerminalPersistence: jest.fn(), + }; + const retire = createChannelResponseRetirement({ + storage, + recovery: { recoverResponse: jest.fn() }, + retireResponse: createResponseRetirement({ storage, jobs }), + }); + await expect(retire(run)).resolves.toBe('pending'); + expect(jobs.abortJob).not.toHaveBeenCalled(); + })); + +it('does not treat a cleanup metadata outage as successful retirement', () => + scoped(async () => { + run.cancellationRequested = true; + run.errorCode = 'channel_authority_changed'; + const retireResponse = jest.fn(async () => 'stopped' as const); + const retire = createChannelResponseRetirement({ + storage: { + readCleanupIdentity: async () => { + throw new Error('cleanup store unavailable'); + }, + }, + recovery: { recoverResponse: jest.fn() }, + retireResponse, + }); + await expect(retire(run)).rejects.toThrow('cleanup store unavailable'); + expect(retireResponse).not.toHaveBeenCalled(); + })); diff --git a/packages/api/src/channels/responses.ts b/packages/api/src/channels/responses.ts new file mode 100644 index 0000000000..fd57e9a892 --- /dev/null +++ b/packages/api/src/channels/responses.ts @@ -0,0 +1,167 @@ +import type { + ChannelRunRecord, + ChannelRunStorage, + ResponseStorage, + ResponseOwner, +} from '@librechat/data-schemas'; +import type { ChannelRunSnapshot } from 'librechat-data-provider'; +import type { BackgroundResponseResult } from '~/agents/responses/background'; +import type { Response } from '~/agents/responses/types'; +import { ChannelExecutionError } from './errors'; + +export interface ChannelResponseProjectionDependencies { + storage: Pick; + /** Reads through live source-key, owner, tenant, role and Agent authorization. */ + readResponse: (owner: ResponseOwner, responseId: string) => Promise; + cancelResponse: (owner: ResponseOwner, responseId: string) => Promise; + now?: () => number; +} + +/** Revoked channel authority can retire its exact response without recovering execution authority. */ +export function createChannelResponseRetirement(deps: { + storage: Pick; + recovery: Pick; + retireResponse: (owner: ResponseOwner, responseId: string) => Promise<'stopped' | 'pending'>; + now?: () => number; +}): (run: ChannelRunRecord) => Promise<'stopped' | 'pending'> { + return async (run) => { + if ( + run.execution?.protocol !== 'responses' || + !run.cancellationRequested || + run.errorCode !== 'channel_authority_changed' + ) + throw new ChannelExecutionError('execution_identity_changed'); + if (!run.execution.responseId) { + if (!run.leaseOwner) throw new ChannelExecutionError('execution_identity_changed'); + const recovered = await deps.recovery.recoverResponse( + run.tenantId, + { runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch }, + (deps.now ?? Date.now)(), + ); + if (!recovered) return 'pending'; + run = recovered; + } + const responseId = run.execution?.responseId; + if (!responseId) return 'pending'; + const owner = { + userId: run.principal.userId, + tenantId: run.tenantId === '_' ? undefined : run.tenantId, + }; + const stored = await deps.storage.readCleanupIdentity(owner, responseId); + if ( + stored && + (stored.responseId !== responseId || + stored.userId !== owner.userId || + (stored.tenantId || undefined) !== (owner.tenantId || undefined) || + stored.agentId !== run.agentId || + stored.requestKey !== `channel:${run.runId}` || + stored.conversationId !== run.execution!.conversationId || + stored.parentMessageId !== run.execution!.parentMessageId || + (run.execution!.apiKeyId != null && stored.sourceKeyId !== run.execution!.apiKeyId)) + ) + throw new ChannelExecutionError('execution_identity_changed'); + // The private retirement owner resolves missing rows from cleanup/job identities; + // absence here never proves that a provider stopped. + return deps.retireResponse(owner, responseId); + }; +} + +function snapshotStatus(response: Response): ChannelRunSnapshot['status'] { + switch (response.status) { + case 'completed': + return 'completed'; + case 'cancelled': + return 'cancelled'; + case 'failed': + case 'incomplete': + return 'failed'; + default: + return response.required_action ? 'requires_action' : 'running'; + } +} + +/** Delivery projection only: never starts an Agent, rebuilds a transcript or infers completion. */ +export function createChannelResponseProjection(deps: ChannelResponseProjectionDependencies) { + return async ( + run: ChannelRunRecord, + ): Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }> => { + const execution = run.execution; + if (execution?.protocol !== 'responses' || !execution.responseId) + throw new ChannelExecutionError('execution_not_prepared'); + const owner = { + userId: run.principal.userId, + tenantId: run.tenantId === '_' ? undefined : run.tenantId, + }; + const stored = await deps.storage.read(owner, execution.responseId); + if ( + !stored || + stored.agentId !== run.agentId || + stored.requestKey !== `channel:${run.runId}` || + stored.conversationId !== execution.conversationId || + stored.parentMessageId !== execution.parentMessageId || + (execution.apiKeyId != null && stored.sourceKeyId !== execution.apiKeyId) || + (execution.generationCreatedAt != null && + stored.generationCreatedAt !== execution.generationCreatedAt) + ) + throw new ChannelExecutionError('execution_identity_changed'); + const result = await (run.cancellationRequested ? deps.cancelResponse : deps.readResponse)( + owner, + execution.responseId, + ); + if (!result.ok) throw new ChannelExecutionError('execution_identity_changed'); + const response = result.response; + if ( + response.id !== execution.responseId || + response.model !== run.agentId || + !response.background || + !response.store + ) + throw new ChannelExecutionError('execution_identity_changed'); + const text: string[] = []; + for (const item of response.output) { + if (item.type !== 'message' || item.role !== 'assistant') continue; + for (const content of item.content) { + if (content.type === 'output_text') text.push(content.text); + } + } + const terminal = ['completed', 'failed', 'incomplete', 'cancelled'].includes(response.status); + const status = snapshotStatus(response); + const action = response.required_action; + return { + generationCreatedAt: stored.generationCreatedAt, + snapshot: { + responseId: response.id, + status, + ...(response.recovery ? { recovery: response.recovery } : {}), + text: terminal ? text.join('') : '', + elapsedMs: Math.max( + 0, + (response.completed_at == null + ? (deps.now ?? Date.now)() + : response.completed_at * 1000) - run.receivedAt, + ), + ...(response.accounting ? { accounting: response.accounting } : {}), + ...(response.usage + ? { + usage: { + inputTokens: response.usage.input_tokens, + outputTokens: response.usage.output_tokens, + totalTokens: response.usage.total_tokens, + cachedTokens: response.usage.input_tokens_details.cached_tokens, + reasoningTokens: response.usage.output_tokens_details.reasoning_tokens, + }, + } + : {}), + ...(!terminal && action + ? { + pendingAction: { + actionId: action.action_id, + kind: action.kind === 'question' ? ('ask_user_question' as const) : action.kind, + expiresAt: action.expires_at * 1000, + }, + } + : {}), + }, + }; + }; +} diff --git a/packages/api/src/channels/runStorage.spec.ts b/packages/api/src/channels/runStorage.spec.ts new file mode 100644 index 0000000000..391368f2cc --- /dev/null +++ b/packages/api/src/channels/runStorage.spec.ts @@ -0,0 +1,1267 @@ +import mongoose from 'mongoose'; +import { MongoMemoryReplSet } from 'mongodb-memory-server'; +import { + createModels, + createMethods, + createChannelRunStorage, + createResponseStorage, + tenantStorage, + runAsSystem, +} from '@librechat/data-schemas'; +import type { + ChannelRunStorage, + ChannelRunEnrollment, + ChannelRunRecord, +} from '@librechat/data-schemas'; +import type { + ChannelRunFence, + ChannelDeliveryPlan, + ChannelRunSummary, +} from 'librechat-data-provider'; +import { + createBackgroundResponseQueue, + createBackgroundResponseEnvelope, +} from '~/agents/responses/queue'; +import { buildPendingBackgroundResponse } from '~/agents/responses/background'; +import { createResponseRetirement } from '~/agents/responses/retirement'; +import { createAgentTriggerService } from '~/agents/triggers/service'; +import { createChannelResponseRetirement } from './responses'; + +let mongo: MongoMemoryReplSet; +let storage: ChannelRunStorage; +const tenantId = 'channel-run-test'; +const scope = (work: () => Promise) => tenantStorage.run({ tenantId }, work); +const fence = (run: ChannelRunRecord): ChannelRunFence => ({ + runId: run.runId, + owner: run.leaseOwner!, + epoch: run.leaseEpoch, +}); +const enrollment = (eventId = 'Ev1', userId = 'owner'): ChannelRunEnrollment => ({ + tenantId, + channelId: 'channel', + agentId: 'agent', + principal: { userId, channelRevision: 'v1' }, + admission: { + eventId, + externalChannelId: 'C1', + externalThreadKey: 'C1:1.001', + externalMessageKey: `C1:${eventId}`, + actor: { teamId: 'T1', appId: 'A1', externalUserId: userId }, + input: 'hello', + }, +}); +const plan: ChannelDeliveryPlan = { + revision: 'r1', + rendererVersion: 'v1', + parts: [ + { operationId: 'part-1', kind: 'message', required: true, text: 'first' }, + { operationId: 'part-2', kind: 'message', required: true, text: 'second' }, + ], +}; + +beforeAll(async () => { + mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } }); + await mongoose.connect(mongo.getUri(), { autoIndex: false }); + createModels(mongoose); + await Promise.all([mongoose.models.ChannelRun.init(), mongoose.models.ChannelLane.init()]); + storage = createChannelRunStorage(mongoose); +}); +afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); +}); +beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.ChannelRun.deleteMany({}); + await mongoose.models.ChannelLane.deleteMany({}); + await mongoose.models.AgentChannel.deleteMany({}); + await mongoose.models.ChannelScope.deleteMany({}); + await mongoose.models.ResponseScope.deleteMany({}); + await mongoose.models.StoredResponse.deleteMany({}); + await createMethods(mongoose).createAgentChannel({ + tenantId, + channelId: 'channel', + agentId: 'agent', + createdBy: 'channel-owner', + type: 'slack', + status: 'active', + credentialMode: 'shared', + revision: 'v1', + slack: { teamId: 'T1', appId: 'A1', botUserId: 'B1' }, + }); + }); +}); + +async function bindTestResponse(run: ChannelRunRecord, responseId = 'response-1', at = 102) { + const current = (await storage.read(run.tenantId, run.channelId, run.runId))!; + const execution = current.execution!; + await mongoose.models.StoredResponse.create({ + tenantId: run.tenantId === '_' ? undefined : run.tenantId, + sourceKeyId: execution.apiKeyId, + userId: run.principal.userId, + responseId, + requestKey: `channel:${run.runId}`, + fingerprint: 'request', + agentId: run.agentId, + requestJSON: JSON.stringify({ + model: run.agentId, + input: run.admission.input, + background: true, + store: true, + ...(execution.parentMessageId === '00000000-0000-0000-0000-000000000000' + ? {} + : { previous_response_id: execution.parentMessageId }), + }), + conversationId: execution.conversationId, + parentMessageId: execution.parentMessageId, + status: 'queued', + revision: 0, + }); + return storage.bindResponse(run.tenantId, fence(run), responseId, at); +} + +async function ready() { + await storage.admit(enrollment(), 100); + const claimed = await storage.claim(tenantId, 'channel', 'worker-a', 100, 1000); + expect(claimed).not.toBeNull(); + const prepared = await storage.prepareExecution(tenantId, fence(claimed!), 101); + const run = await bindTestResponse(prepared); + await storage.saveSnapshot( + tenantId, + fence(run), + { responseId: 'response-1', status: 'completed', text: 'first second', elapsedMs: 10 }, + 102, + ); + return storage.savePlan(tenantId, fence(run), plan, 103); +} + +it('deduplicates admission, rejects changed payload and allocates ordered sequence', () => + scope(async () => { + const one = await storage.admit(enrollment(), 100); + const replay = await storage.admit(enrollment(), 101); + expect(replay.duplicate).toBe(true); + expect(replay.run.runId).toBe(one.run.runId); + const altered = enrollment(); + altered.admission.input = 'different'; + await expect(storage.admit(altered, 102)).rejects.toMatchObject({ code: 'event_conflict' }); + const two = await storage.admit(enrollment('Ev2'), 102); + expect(two.run.sequence).toBe(one.run.sequence + 1); + })); + +async function savedResponse( + run: ChannelRunRecord, + overrides: { + tenantId?: string; + userId?: string; + agentId?: string; + requestKey?: string; + input?: string; + previous_response_id?: string; + responseId?: string; + sourceKeyId?: string; + } = {}, +) { + const responseStorage = createResponseStorage(mongoose); + const result = await responseStorage.reserve({ + tenantId: overrides.tenantId ?? tenantId, + userId: overrides.userId ?? run.principal.userId, + agentId: overrides.agentId ?? run.agentId, + responseId: overrides.responseId ?? 'resp_bound', + requestKey: overrides.requestKey ?? `channel:${run.runId}`, + sourceKeyId: overrides.sourceKeyId, + fingerprint: `fingerprint-${overrides.responseId ?? 'bound'}`, + requestJSON: JSON.stringify({ + model: overrides.agentId ?? run.agentId, + input: overrides.input ?? run.admission.input, + background: true, + store: true, + previous_response_id: overrides.previous_response_id, + }), + conversationId: 'response-owned-conversation', + parentMessageId: '00000000-0000-0000-0000-000000000000', + }); + if (!result.ok) throw new Error('Response reservation failed'); + return result.record; +} + +describe('Responses execution binding', () => { + beforeEach(async () => { + await runAsSystem(async () => { + await mongoose.models.StoredResponse.deleteMany({}); + }); + }); + async function prepared() { + await storage.admit(enrollment(), 100); + const claimed = (await storage.claim(tenantId, 'channel', 'worker', 100, 1000))!; + return storage.prepareExecution(tenantId, fence(claimed), 101); + } + async function issued(cancel = true) { + const preparedRun = await prepared(); + const run = await storage.attachResponseCredential( + tenantId, + fence(preparedRun), + { apiKeyId: 'issued', encryptedApiKey: 'ciphertext' }, + 102, + ); + if (cancel) await storage.requestCancel(run, run.runId); + return run; + } + it('holds a cancelled issued grant until the exact late response exists, without writing that response', () => + scope(async () => { + const run = await issued(); + expect(await storage.recoverResponse(tenantId, fence(run), 103)).toBeNull(); + const held = await storage.read(tenantId, 'channel', run.runId); + expect(held).toMatchObject({ state: 'executing', cancellationRequested: true }); + expect(held?.execution?.responseId).toBeUndefined(); + expect(held?.snapshot).toBeUndefined(); + const response = await savedResponse(run, { sourceKeyId: 'issued' }); + const recovered = await storage.recoverResponse(tenantId, fence(run), 104); + expect(recovered?.execution).toMatchObject({ responseId: response.responseId }); + expect(await createResponseStorage(mongoose).read(response, response.responseId)).toEqual( + response, + ); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + })); + it('replays concurrent recovery and a lost binding acknowledgement without another creation', () => + scope(async () => { + const run = await issued(); + const response = await savedResponse(run, { sourceKeyId: 'issued' }); + const recovered = await Promise.all( + Array.from({ length: 3 }, () => storage.recoverResponse(tenantId, fence(run), 103)), + ); + expect(recovered.map((value) => value?.execution?.responseId)).toEqual( + Array(3).fill(response.responseId), + ); + expect((await storage.recoverResponse(tenantId, fence(run), 104))?.execution).toEqual( + recovered[0]?.execution, + ); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + })); + it('recovers an already committed bind when its acknowledgement was lost', () => + scope(async () => { + const run = await issued(); + const response = await savedResponse(run, { sourceKeyId: 'issued' }); + const bind = async () => { + await storage.bindResponse(tenantId, fence(run), response.responseId, 103); + throw new Error('Bind acknowledgement lost'); + }; + await expect(bind()).rejects.toThrow('Bind acknowledgement lost'); + expect( + (await storage.recoverResponse(tenantId, fence(run), 104))?.execution?.responseId, + ).toBe(response.responseId); + expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(1); + })); + it('does not discover the same request identity stored in another tenant', () => + scope(async () => { + const run = await issued(); + await tenantStorage.run({ tenantId: 'foreign-tenant' }, () => + savedResponse(run, { tenantId: 'foreign-tenant', sourceKeyId: 'issued' }), + ); + expect(await storage.recoverResponse(tenantId, fence(run), 103)).toBeNull(); + })); + it.each([ + { userId: 'other-owner' }, + { agentId: 'other-agent' }, + { requestKey: 'other-request' }, + { sourceKeyId: 'other-key' }, + ])('does not recover a foreign response: %j', (overrides) => + scope(async () => { + const run = await issued(); + const response = await savedResponse(run, { sourceKeyId: 'issued', ...overrides }); + expect(await storage.recoverResponse(tenantId, fence(run), 103)).toBeNull(); + expect(await createResponseStorage(mongoose).read(response, response.responseId)).toEqual( + response, + ); + }), + ); + it.each([{ input: 'different' }, { previous_response_id: 'resp_foreign' }])( + 'rejects canonical identity with a mismatched admitted request: %j', + (overrides) => + scope(async () => { + const run = await issued(); + await savedResponse(run, { sourceKeyId: 'issued', ...overrides }); + await expect(storage.recoverResponse(tenantId, fence(run), 103)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + expect( + (await storage.read(tenantId, 'channel', run.runId))?.execution?.responseId, + ).toBeUndefined(); + }), + ); + it('requires cancellation, matching tenant and the current sender epoch inside recovery', () => + scope(async () => { + const run = await issued(false); + await savedResponse(run, { sourceKeyId: 'issued' }); + await expect(storage.recoverResponse(tenantId, fence(run), 103)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + await storage.requestCancel(run, run.runId); + await expect(storage.recoverResponse('other-tenant', fence(run), 103)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + await expect( + storage.recoverResponse(tenantId, { ...fence(run), epoch: 0 }, 103), + ).rejects.toMatchObject({ code: 'stale_fence' }); + await expect(storage.recoverResponse(tenantId, fence(run), 1100)).rejects.toMatchObject({ + code: 'stale_fence', + }); + })); + it('propagates storage failure rather than treating it as absent creation', () => + scope(async () => { + const run = await issued(); + const failure = jest + .spyOn(mongoose.models.StoredResponse, 'findOne') + .mockImplementationOnce(() => { + throw new Error('Storage unavailable'); + }); + await expect(storage.recoverResponse(tenantId, fence(run), 103)).rejects.toThrow( + 'Storage unavailable', + ); + failure.mockRestore(); + expect(await storage.recoverResponse(tenantId, fence(run), 104)).toBeNull(); + })); + it('recovers revoked queued creation after a lost queue ACK and retires only the canonical response', () => + scope(async () => { + await storage.admit(enrollment('queue-stop', '000000000000000000000001'), 100); + const claimed = (await storage.claim(tenantId, 'channel', 'worker', 100, 1000))!; + await storage.prepareExecution(tenantId, fence(claimed), 101); + const run = await storage.attachResponseCredential( + tenantId, + fence(claimed), + { apiKeyId: 'issued', encryptedApiKey: 'ciphertext' }, + 102, + ); + const responses = createResponseStorage(mongoose); + const reserved = await savedResponse(run, { sourceKeyId: 'issued' }); + const methods = createMethods(mongoose); + const service = createAgentTriggerService({ + methods: { + ...methods, + getAgentTriggerDeliveryBatch: (delivery) => + methods.getAgentTriggerDeliveryBatch({ + id: delivery.id, + batchMemberIds: delivery.batchMemberIds?.map( + (id) => new mongoose.Types.ObjectId(String(id)), + ), + }), + }, + }); + const execute = jest.fn(async () => ({ status: 'started' as const })); + const queue = createBackgroundResponseQueue({ + storage: responses, + enqueue: async (...args) => { + await service.enqueue(...args); + throw new Error('Queue acknowledgement lost'); + }, + authorize: async () => true, + execute, + }); + const attached = await responses.attachDelivery( + reserved, + reserved.responseId, + queue.deliveryKey(reserved), + ); + if (!attached.ok) throw new Error('Attach failed'); + try { + await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3088 } }); + await expect(queue.enqueue(attached.record)).rejects.toThrow('Queue acknowledgement lost'); + expect(await service.getDelivery(attached.record.deliveryKey!)).toMatchObject({ + status: 'capability_pending', + }); + const cleanup = await storage.claimAuthorityLost(run, 'cleanup', 1101, 1000); + expect(cleanup).not.toBeNull(); + const jobs = { + getJob: jest.fn(async () => undefined), + abortJob: jest.fn(), + finishProviderTerminalPersistence: jest.fn(async () => false), + }; + const retire = createChannelResponseRetirement({ + storage: responses, + recovery: storage, + retireResponse: createResponseRetirement({ storage: responses, jobs, now: () => 1103 }), + now: () => 1102, + }); + const publication = jest + .spyOn(responses, 'publish') + .mockRejectedValueOnce(new Error('Publication unavailable')); + await expect(retire(cleanup!)).rejects.toThrow('Publication unavailable'); + expect(await responses.read(reserved, reserved.responseId)).toMatchObject({ + status: 'queued', + }); + await expect(retire(cleanup!)).resolves.toBe('stopped'); + await expect(retire(cleanup!)).resolves.toBe('stopped'); + publication.mockRestore(); + expect(await responses.read(reserved, reserved.responseId)).toMatchObject({ + status: 'cancelled', + }); + const dispatcher = queue.sourceDispatchers.get('responses')!; + await expect( + dispatcher.dispatch(createBackgroundResponseEnvelope(attached.record)), + ).resolves.toMatchObject({ status: 'settled' }); + expect(execute).not.toHaveBeenCalled(); + expect(jobs.abortJob).not.toHaveBeenCalled(); + expect((await storage.read(tenantId, 'channel', run.runId))?.snapshot).toBeUndefined(); + } finally { + await service.stop(); + } + })); + it('retains a recovered admitted response when the provider drain proof is missing', () => + scope(async () => { + const run = await issued(); + const responses = createResponseStorage(mongoose); + const response = await savedResponse(run, { sourceKeyId: 'issued' }); + const publication = await responses.publish( + response, + response.responseId, + response.revision, + { + generationCreatedAt: 103, + status: 'in_progress', + responseJSON: JSON.stringify({ + ...buildPendingBackgroundResponse(response), + status: 'in_progress', + }), + }, + ); + expect(publication.ok).toBe(true); + const cleanup = await storage.claimAuthorityLost(run, 'cleanup', 1101, 1000); + const jobs = { + getJob: jest.fn(async () => undefined), + abortJob: jest.fn(), + finishProviderTerminalPersistence: jest.fn(async () => false), + }; + const retire = createChannelResponseRetirement({ + storage: responses, + recovery: storage, + retireResponse: createResponseRetirement({ storage: responses, jobs, now: () => 1103 }), + now: () => 1102, + }); + await expect(retire(cleanup!)).resolves.toBe('pending'); + expect(jobs.abortJob).not.toHaveBeenCalled(); + expect(await responses.read(response, response.responseId)).toMatchObject({ + status: 'in_progress', + }); + const held = await storage.read(tenantId, 'channel', run.runId); + expect(held?.execution).toMatchObject({ + responseId: response.responseId, + generationCreatedAt: 103, + }); + expect(held?.state).toBe('executing'); + expect(held?.snapshot).toBeUndefined(); + })); + it('confirms unstarted cancellation only while no credential or provider admission can exist', () => + scope(async () => { + const run = await prepared(); + await storage.requestCancel(run, run.runId); + const cancelled = await storage.finishUnstartedResponseCancellation( + tenantId, + fence(run), + 102, + ); + expect(cancelled.state).toBe('delivering'); + expect(cancelled.snapshot).toMatchObject({ status: 'cancelled' }); + expect(cancelled.execution?.apiKeyId).toBeUndefined(); + await expect( + storage.attachResponseCredential( + tenantId, + fence(run), + { apiKeyId: 'late-key', encryptedApiKey: 'ciphertext' }, + 103, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + it('does not claim an already issued credential could never have executed', () => + scope(async () => { + const run = await prepared(); + await storage.attachResponseCredential( + tenantId, + fence(run), + { apiKeyId: 'issued', encryptedApiKey: 'ciphertext' }, + 102, + ); + await storage.requestCancel(run, run.runId); + await expect( + storage.finishUnstartedResponseCancellation(tenantId, fence(run), 103), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + it('binds the actual owner-scoped response idempotently', () => + scope(async () => { + const run = await prepared(); + const response = await savedResponse(run); + const bound = await storage.bindResponse(tenantId, fence(run), response.responseId, 102); + expect(bound.execution).toMatchObject({ + protocol: 'responses', + responseId: response.responseId, + conversationId: response.conversationId, + parentMessageId: response.parentMessageId, + }); + const replayed = await storage.bindResponse(tenantId, fence(run), response.responseId, 103); + expect(replayed.execution).toEqual(bound.execution); + expect(bound.execution?.deliveryKey).toBeUndefined(); + expect(bound.execution?.enqueueAttemptedAt).toBeUndefined(); + })); + it.each([ + { userId: 'other-owner' }, + { agentId: 'other-agent' }, + { requestKey: 'other-event' }, + { input: 'different input' }, + { previous_response_id: 'resp_other_history' }, + ])('rejects a response from another execution boundary: %j', (overrides) => + scope(async () => { + const run = await prepared(); + const response = await savedResponse(run, overrides); + await expect( + storage.bindResponse(tenantId, fence(run), response.responseId, 102), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + expect( + (await storage.read(tenantId, 'channel', run.runId))?.execution?.protocol, + ).toBeUndefined(); + }), + ); + it('cannot switch an already-dispatched legacy run to Responses', () => + scope(async () => { + const run = await prepared(); + const response = await savedResponse(run); + // Seed historical state directly; production only dispatches through Responses. + await mongoose.models.ChannelRun.updateOne( + { tenantId, runId: run.runId }, + { + $set: { 'execution.deliveryKey': 'legacy-trigger', 'execution.enqueueAttemptedAt': 102 }, + }, + ); + await expect( + storage.bindResponse(tenantId, fence(run), response.responseId, 103), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + it('requires the current sender fence', () => + scope(async () => { + const run = await prepared(); + const response = await savedResponse(run); + await expect( + storage.bindResponse( + tenantId, + { ...fence(run), epoch: run.leaseEpoch + 1 }, + response.responseId, + 102, + ), + ).rejects.toMatchObject({ code: 'stale_fence' }); + })); +}); + +it('fences generation snapshots and releases only the cancelled lane without advancing its cursor', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const other = await storage.admit(enrollment('other-actor', 'other'), 101); + const next = await storage.admit(enrollment('next'), 102); + const claimed = (await storage.claim(tenantId, 'channel', 'worker', 103, 1000))!; + const run = await storage.prepareExecution(tenantId, fence(claimed), 104); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'running', text: '', elapsedMs: 1 }, + 105, + 106, + ); + await expect( + storage.saveSnapshot( + tenantId, + fence(run), + { status: 'running', text: '', elapsedMs: 1 }, + 108, + 109, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'cancelled', text: 'Cancelled.', elapsedMs: 1 }, + 108, + 106, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'stop', + rendererVersion: 'v1', + parts: [{ operationId: 'notice', kind: 'message', required: true, text: 'Cancelled.' }], + }, + 109, + ); + await storage.beginDelivery(tenantId, fence(run), 'notice', 110); + await storage.settleDelivery( + tenantId, + fence(run), + { + operationId: 'notice', + outcome: 'confirmed', + attempts: 1, + externalMessageKey: 'C1:notice', + }, + 111, + ); + expect((await storage.commitDelivery(tenantId, fence(run), 112)).state).toBe('cancelled'); + const lane = await mongoose.models.ChannelLane.findOne({ tenantId, laneId: run.laneId }).lean(); + expect(lane).not.toHaveProperty('activeRunId'); + expect(lane).not.toHaveProperty('responseId'); + expect(lane).not.toHaveProperty('conversationId'); + const following = (await storage.claim(tenantId, 'channel', 'next-worker', 113, 1000))!; + expect([other.run.runId, next.run.runId]).toContain(following.runId); + const another = (await storage.claim(tenantId, 'channel', 'another-worker', 114, 1000))!; + expect([following.runId, another.runId]).toContain(next.run.runId); + expect([following.runId, another.runId]).toContain(other.run.runId); + })); + +it('serializes simultaneous duplicate ingress without duplicate executions', () => + scope(async () => { + const results = await Promise.all( + Array.from({ length: 5 }, () => storage.admit(enrollment(), 100)), + ); + expect(new Set(results.map(({ run }) => run.runId)).size).toBe(1); + expect(results.filter(({ duplicate }) => !duplicate)).toHaveLength(1); + })); + +it('holds one actor lane while allowing another principal and fences expired workers', () => + scope(async () => { + await storage.admit(enrollment(), 100); + await storage.admit(enrollment('Ev2'), 101); + await storage.admit(enrollment('Ev3', 'other'), 102); + const first = (await storage.claim(tenantId, 'channel', 'one', 103, 10))!; + const second = (await storage.claim(tenantId, 'channel', 'two', 104, 10))!; + expect(second.principal.userId).toBe('other'); + expect(await storage.claim(tenantId, 'channel', 'three', 105, 10)).toBeNull(); + const replacement = (await storage.claim(tenantId, 'channel', 'three', 114, 10))!; + expect(replacement.runId).toBe(first.runId); + expect(replacement.leaseEpoch).toBe(first.leaseEpoch + 1); + expect(await storage.renew(tenantId, fence(first), 114, 10)).toBe(false); + await expect(storage.prepareExecution(tenantId, fence(first), 114)).rejects.toMatchObject({ + code: 'stale_fence', + }); + })); + +it('preserves the exact execution identity across process-owner replacement', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const first = (await storage.claim(tenantId, 'channel', 'one', 100, 10))!; + const executing = await storage.prepareExecution(tenantId, fence(first), 101); + const replacement = (await storage.claim(tenantId, 'channel', 'two', 111, 20))!; + const resumed = await storage.prepareExecution(tenantId, fence(replacement), 112); + expect(resumed.execution).toEqual(executing.execution); + })); + +async function waitingResponse() { + await storage.admit(enrollment(), 100); + const claimed = (await storage.claim(tenantId, 'channel', 'worker', 100, 60_000))!; + const prepared = await storage.prepareExecution(tenantId, fence(claimed), 101); + await mongoose.models.ChannelRun.updateOne( + { tenantId, runId: prepared.runId }, + { + $set: { + execution: { ...prepared.execution, protocol: 'responses', responseId: 'response-1' }, + }, + }, + ); + return (await storage.read(tenantId, 'channel', prepared.runId))!; +} + +it('yields only the polling lease, invalidates its fence, and reclaims one exact Response after cooldown', () => + scope(async () => { + const run = await waitingResponse(); + await storage.admit(enrollment('next'), 102); + await storage.admit(enrollment('other', 'other'), 103); + expect( + (await storage.claim(tenantId, 'channel', 'other-worker', 104, 60_000))?.principal.userId, + ).toBe('other'); + await storage.requestCancel(run, run.runId); + expect(await storage.yieldLease(tenantId, 'channel', fence(run), 105)).toBe(true); + const yielded = (await storage.read(tenantId, 'channel', run.runId))!; + expect(yielded.leaseOwner).toBeUndefined(); + expect(yielded).toMatchObject({ + leaseEpoch: run.leaseEpoch, + leaseExpiresAt: 1105, + claimCheckedAt: run.claimCheckedAt, + execution: run.execution, + cancellationRequested: true, + }); + const lane = await mongoose.models.ChannelLane.findOne({ tenantId, laneId: run.laneId }).lean(); + expect(lane).toMatchObject({ activeRunId: run.runId }); + expect(await storage.renew(tenantId, fence(run), 106, 60_000)).toBe(false); + await expect(storage.prepareExecution(tenantId, fence(run), 106)).rejects.toMatchObject({ + code: 'stale_fence', + }); + expect(await storage.claim(tenantId, 'channel', 'too-soon', 1104, 60_000)).toBeNull(); + const claims = await Promise.all( + ['replica-a', 'replica-b'].map((owner) => + storage.claim(tenantId, 'channel', owner, 1105, 60_000), + ), + ); + const winners = claims.filter((claim) => claim !== null); + expect(winners).toHaveLength(1); + expect(winners[0]).toMatchObject({ + runId: run.runId, + leaseEpoch: run.leaseEpoch + 1, + execution: run.execution, + cancellationRequested: true, + }); + expect(await storage.yieldLease(tenantId, 'channel', fence(run), 1106)).toBe(false); + })); + +it.each(['tenant', 'channel', 'owner', 'epoch', 'expired'])( + 'rejects a %s-mismatched polling lease yield', + (kind) => + scope(async () => { + const run = await waitingResponse(); + const selected = { + ...fence(run), + ...(kind === 'owner' ? { owner: 'foreign' } : {}), + ...(kind === 'epoch' ? { epoch: run.leaseEpoch + 1 } : {}), + }; + expect( + await storage.yieldLease( + kind === 'tenant' ? 'foreign' : tenantId, + kind === 'channel' ? 'foreign' : 'channel', + selected, + kind === 'expired' ? 60_100 : 102, + ), + ).toBe(false); + expect((await storage.read(tenantId, 'channel', run.runId))?.leaseOwner).toBe('worker'); + }), +); + +it.each([ + 'plan', + 'inflight-progress', + 'unknown-progress', + 'inflight-receipt', + 'unknown-receipt', + 'unbound', + 'legacy', +] as const)('does not yield unsafe %s work', (kind) => + scope(async () => { + const run = await waitingResponse(); + const patch = { + plan: { plan }, + 'inflight-progress': { progress: { outcome: 'inflight' } }, + 'unknown-progress': { progress: { outcome: 'unknown' } }, + 'inflight-receipt': { + receipts: [{ operationId: 'part-1', outcome: 'inflight', attempts: 1 }], + }, + 'unknown-receipt': { receipts: [{ operationId: 'part-1', outcome: 'unknown', attempts: 1 }] }, + unbound: { execution: run.execution }, + legacy: { execution: { ...run.execution, protocol: 'legacy' } }, + }[kind]; + await mongoose.models.ChannelRun.updateOne({ tenantId, runId: run.runId }, { $set: patch }); + if (kind === 'unbound') + await mongoose.models.ChannelRun.updateOne( + { tenantId, runId: run.runId }, + { $unset: { 'execution.responseId': 1 } }, + ); + expect(await storage.yieldLease(tenantId, 'channel', fence(run), 102)).toBe(false); + }), +); + +it('cannot renew or yield again after a concurrent successful yield', () => + scope(async () => { + const run = await waitingResponse(); + const results = await Promise.all([ + storage.yieldLease(tenantId, 'channel', fence(run), 102), + storage.yieldLease(tenantId, 'channel', fence(run), 102), + ]); + expect(results.filter(Boolean)).toHaveLength(1); + expect(await storage.renew(tenantId, fence(run), 103, 60_000)).toBe(false); + })); + +it('retains ambiguous remote writes and never reopens them for an automatic repost', () => + scope(async () => { + const run = await ready(); + await storage.beginDelivery(tenantId, fence(run), 'part-1', 104); + const replacement = (await storage.claim(tenantId, 'channel', 'replacement', 1101, 1000))!; + expect(replacement.receipts[0].outcome).toBe('unknown'); + await expect( + storage.beginDelivery(tenantId, fence(replacement), 'part-1', 1102), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await expect(storage.commitDelivery(tenantId, fence(replacement), 1102)).rejects.toMatchObject({ + code: 'delivery_incomplete', + }); + })); + +it('commits all confirmed reply mappings and cursor atomically, then releases FIFO', () => + scope(async () => { + const run = await ready(); + await storage.admit(enrollment('Ev2'), 104); + await expect(storage.beginDelivery(tenantId, fence(run), 'part-2', 104)).rejects.toMatchObject({ + code: 'delivery_incomplete', + }); + await expect(storage.commitDelivery(tenantId, fence(run), 104)).rejects.toMatchObject({ + code: 'delivery_incomplete', + }); + for (const [index, part] of plan.parts.entries()) { + await storage.beginDelivery(tenantId, fence(run), part.operationId, 105 + index); + await storage.settleDelivery( + tenantId, + fence(run), + { + operationId: part.operationId, + attempts: 1, + outcome: 'confirmed', + externalMessageKey: `C1:reply-${index}`, + }, + 107 + index, + ); + } + const committed = await storage.commitDelivery(tenantId, fence(run), 110); + expect(committed.state).toBe('delivered'); + expect((await storage.commitDelivery(tenantId, fence(run), 111)).runId).toBe(run.runId); + expect((await storage.findDeliveredReply(run, 'C1:reply-1'))?.runId).toBe(run.runId); + const next = (await storage.claim(tenantId, 'channel', 'next', 112, 1000))!; + const execution = await storage.prepareExecution(tenantId, fence(next), 113); + expect(execution.execution?.parentMessageId).toBe('response-1'); + expect(execution.execution?.conversationId).toBe(run.execution?.conversationId); + })); + +it('honors persisted retry time and immutable plans', () => + scope(async () => { + const run = await ready(); + await expect( + storage.savePlan(tenantId, fence(run), { ...plan, rendererVersion: 'changed' }, 104), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.beginDelivery(tenantId, fence(run), 'part-1', 105); + await storage.settleDelivery( + tenantId, + fence(run), + { operationId: 'part-1', attempts: 1, outcome: 'retryable', retryAt: 200 }, + 106, + ); + await expect(storage.beginDelivery(tenantId, fence(run), 'part-1', 199)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + expect( + (await storage.beginDelivery(tenantId, fence(run), 'part-1', 200)).receipts[0].attempts, + ).toBe(2); + })); + +it('enforces tenant and initiating actor boundaries for reads and cancellation', () => + scope(async () => { + const { run } = await storage.admit(enrollment(), 100); + expect( + await storage.requestCancel( + { + ...run, + admission: { + ...run.admission, + actor: { ...run.admission.actor, externalUserId: 'intruder' }, + }, + }, + run.runId, + ), + ).toBe(false); + expect(await storage.requestCancel(run, run.runId)).toBe(true); + await tenantStorage.run({ tenantId: 'different' }, async () => { + expect(await storage.read(tenantId, 'channel', run.runId)).toBeNull(); + expect(await storage.claim(tenantId, 'channel', 'intruder', 100, 100)).toBeNull(); + }); + })); + +it('native Stop lookup excludes a newer claim, even when its message was already queued', () => + scope(async () => { + const input = enrollment(); + const { run } = await storage.admit(input, 100); + await storage.claim(tenantId, 'channel', 'worker', 200, 100); + const lookup = (at: number, actor = 'owner') => + storage.readActiveThread( + { + ...input, + admission: { + ...input.admission, + actor: { ...input.admission.actor, externalUserId: actor }, + }, + }, + at, + ); + expect(await lookup(199)).toBeNull(); + expect((await lookup(200))?.runId).toBe(run.runId); + expect(await lookup(200, 'foreign')).toBeNull(); + // Reclaimed work rejects the old Stop instead of guessing its intent. + await storage.claim(tenantId, 'channel', 'replacement', 301, 100); + expect(await lookup(200)).toBeNull(); + expect((await lookup(301))?.runId).toBe(run.runId); + })); + +it('persists progress handles and monotonic update intent, freezing ambiguous writes after replacement', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const run = (await storage.claim(tenantId, 'channel', 'one', 100, 100))!; + const progress = { revision: 1, rendererVersion: 'v1', text: 'Request received' }; + await storage.planProgress(tenantId, fence(run), progress, 101); + await storage.beginProgress(tenantId, fence(run), 1, 102); + await expect( + storage.planProgress(tenantId, fence(run), { ...progress, revision: 2 }, 103), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.settleProgress( + tenantId, + fence(run), + { revision: 1, outcome: 'confirmed', attempts: 1, externalMessageKey: 'C1:status' }, + 104, + ); + const update = await storage.planProgress( + tenantId, + fence(run), + { ...progress, revision: 2, text: 'Working' }, + 105, + ); + expect(update.progress?.externalMessageKey).toBe('C1:status'); + await storage.beginProgress(tenantId, fence(run), 2, 106); + const recovered = (await storage.claim(tenantId, 'channel', 'two', 201, 100))!; + expect(recovered.progress).toMatchObject({ + revision: 2, + outcome: 'unknown', + externalMessageKey: 'C1:status', + }); + await expect(storage.beginProgress(tenantId, fence(recovered), 2, 202)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + await expect( + storage.settleProgress( + tenantId, + fence(run), + { revision: 2, outcome: 'confirmed', attempts: 1, externalMessageKey: 'C1:status' }, + 202, + ), + ).rejects.toMatchObject({ code: 'stale_fence' }); + })); + +it('allows known-handle terminal cleanup before final plan and never creates a new terminal surface', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const run = (await storage.claim(tenantId, 'channel', 'one', 100, 100))!; + await storage.prepareExecution(tenantId, fence(run), 101); + await bindTestResponse(run, 'response'); + await storage.planProgress( + tenantId, + fence(run), + { revision: 1, rendererVersion: 'v1', text: 'Working' }, + 102, + ); + await storage.beginProgress(tenantId, fence(run), 1, 103); + await storage.settleProgress( + tenantId, + fence(run), + { revision: 1, outcome: 'confirmed', attempts: 1, externalMessageKey: 'C1:status' }, + 104, + ); + await storage.saveSnapshot( + tenantId, + fence(run), + { status: 'completed', text: 'Answer', responseId: 'response', elapsedMs: 10 }, + 105, + ); + await storage.planProgress( + tenantId, + fence(run), + { revision: 2, rendererVersion: 'v1', text: 'Completed' }, + 106, + ); + await storage.beginProgress(tenantId, fence(run), 2, 107); + await storage.settleProgress( + tenantId, + fence(run), + { revision: 2, outcome: 'confirmed', attempts: 1, externalMessageKey: 'C1:status' }, + 108, + ); + await storage.savePlan(tenantId, fence(run), plan, 109); + await expect( + storage.planProgress( + tenantId, + fence(run), + { revision: 3, rendererVersion: 'v1', text: 'Late' }, + 110, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('retains the original response credential and refuses replacement after cancellation', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const run = (await storage.claim(tenantId, 'channel', 'one', 100, 100))!; + await storage.prepareExecution(tenantId, fence(run), 101); + const original = { apiKeyId: 'original', encryptedApiKey: 'ciphertext' }; + await storage.attachResponseCredential(tenantId, fence(run), original, 102); + await expect( + storage.attachResponseCredential( + tenantId, + fence(run), + { apiKeyId: 'replacement', encryptedApiKey: 'other' }, + 103, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + expect((await storage.read(tenantId, 'channel', run.runId))?.execution).toMatchObject(original); + await storage.requestCancel(run, run.runId); + await expect( + storage.attachResponseCredential(tenantId, fence(run), original, 104), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + })); + +it('freezes a planned terminal summary inside the fence transaction even for stale concurrent projections', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const run = (await storage.claim(tenantId, 'channel', 'worker', 100, 100))!; + await storage.prepareExecution(tenantId, fence(run), 101); + await bindTestResponse(run, 'response'); + const summary: ChannelRunSummary = { + version: 1, + scope: 'actor_lane', + throughSequence: 1, + includesCurrent: true, + priorRuns: 0, + expectedRuns: 1, + observedRuns: 1, + costUSD: { coverage: 'unavailable', coveredRuns: 0 }, + tokens: { coverage: 'unavailable', coveredRuns: 0 }, + elapsedMs: { coverage: 'complete', coveredRuns: 1, value: 12 }, + }; + const snapshot = { + responseId: 'response', + status: 'completed' as const, + text: 'Answer', + elapsedMs: 12, + summary, + }; + await storage.saveSnapshot(tenantId, fence(run), snapshot, 102); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'summary-plan', + rendererVersion: 'v1', + parts: [{ operationId: 'answer', kind: 'message', required: true, text: 'Answer' }], + }, + 102, + ); + await expect( + storage.saveSnapshot( + tenantId, + fence(run), + { + ...snapshot, + summary: { ...summary, elapsedMs: { coverage: 'unavailable', coveredRuns: 0 } }, + }, + 103, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + const result = await storage.saveSnapshot( + tenantId, + fence(run), + { ...snapshot, summary: undefined }, + 103, + ); + expect(result.snapshot?.summary).toEqual(summary); + const replacement = (await storage.claim(tenantId, 'channel', 'next-worker', 201, 100))!; + await expect(storage.saveSnapshot(tenantId, fence(run), snapshot, 202)).rejects.toMatchObject({ + code: 'stale_fence', + }); + expect(replacement.snapshot?.summary).toEqual(summary); + })); + +it('retains exact upload allocation and stage across a crash without reopening remote creation', () => + scope(async () => { + await storage.admit(enrollment(), 100); + const run = (await storage.claim(tenantId, 'channel', 'one', 100, 100))!; + await storage.prepareExecution(tenantId, fence(run), 101); + await bindTestResponse(run, 'response'); + await storage.saveSnapshot( + tenantId, + fence(run), + { + status: 'completed', + responseId: 'response', + text: 'File', + elapsedMs: 1, + artifacts: [ + { + artifactId: 'opaque', + filename: 'report.txt', + mediaType: 'text/plain', + bytes: 4, + sha256: 'hash', + }, + ], + }, + 102, + ); + await storage.savePlan( + tenantId, + fence(run), + { + revision: 'file-plan', + rendererVersion: 'v1', + parts: [ + { + operationId: 'file', + kind: 'artifact', + artifactId: 'opaque', + required: true, + text: 'Report', + }, + ], + }, + 103, + ); + await storage.beginDelivery(tenantId, fence(run), 'file', 104); + await expect( + storage.checkpointUpload( + tenantId, + fence(run), + 'file', + 1, + { fileId: 'F1', stage: 'uploaded' }, + 105, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.checkpointUpload( + tenantId, + fence(run), + 'file', + 1, + { fileId: 'F1', stage: 'allocated' }, + 106, + ); + await expect( + storage.checkpointUpload( + tenantId, + fence(run), + 'file', + 1, + { fileId: 'F2', stage: 'uploaded' }, + 107, + ), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await storage.checkpointUpload( + tenantId, + fence(run), + 'file', + 1, + { fileId: 'F1', stage: 'uploaded' }, + 108, + ); + const replacement = (await storage.claim(tenantId, 'channel', 'two', 201, 100))!; + expect(replacement.receipts[0]).toMatchObject({ + outcome: 'unknown', + upload: { fileId: 'F1', stage: 'uploaded' }, + }); + await expect( + storage.beginDelivery(tenantId, fence(replacement), 'file', 202), + ).rejects.toMatchObject({ code: 'invalid_transition' }); + await expect( + storage.checkpointUpload( + tenantId, + fence(run), + 'file', + 1, + { fileId: 'F1', stage: 'uploaded' }, + 202, + ), + ).rejects.toMatchObject({ code: 'stale_fence' }); + })); + +it('does not give actor B actor A history or cancellation even when the configured shared principal is identical', () => + scope(async () => { + const runA = await ready(); + const eventB = enrollment('EvB'); + eventB.admission.actor.externalUserId = 'actor-B'; + const { run: runB } = await storage.admit(eventB, 104); + expect(runB.principal.userId).toBe(runA.principal.userId); + expect(runB.laneId).not.toBe(runA.laneId); + expect(await storage.requestCancel(eventB, runA.runId)).toBe(false); + const claimedB = (await storage.claim(tenantId, 'channel', 'worker-B', 105, 1000))!; + expect(claimedB.runId).toBe(runB.runId); + const preparedB = await storage.prepareExecution(tenantId, fence(claimedB), 106); + expect(preparedB.execution?.conversationId).not.toBe(runA.execution?.conversationId); + expect(preparedB.execution?.parentMessageId).toBe('00000000-0000-0000-0000-000000000000'); + })); + +it('bounds each fair scan and persists progress across restarts past a full backlog', () => + scope(async () => { + await storage.admit(enrollment('head'), 100); + await storage.claim(tenantId, 'channel', 'holding-worker', 100, 10_000); + for (let i = 0; i < 105; i++) await storage.admit(enrollment(`queued-${i}`), 101 + i); + const next = await storage.admit(enrollment('actor-b', 'actor-b'), 300); + const reads = jest.spyOn(mongoose.models.ChannelLane, 'findOne'); + const predecessors = jest.spyOn(mongoose.models.ChannelRun, 'exists'); + const writes = jest.spyOn(mongoose.models.ChannelRun, 'updateOne'); + for (let attempt = 0; attempt < 4; attempt++) { + reads.mockClear(); + predecessors.mockClear(); + writes.mockClear(); + storage = createChannelRunStorage(mongoose); + expect(await storage.claim(tenantId, 'channel', 'worker-b', 301 + attempt, 1000)).toBeNull(); + expect(reads.mock.calls.length).toBeLessThanOrEqual(25); + expect(predecessors.mock.calls.length).toBeLessThanOrEqual(25); + expect(writes.mock.calls.length).toBe(25); + } + storage = createChannelRunStorage(mongoose); + expect((await storage.claim(tenantId, 'channel', 'worker-b', 305, 1000))?.runId).toBe( + next.run.runId, + ); + const explanation = await mongoose.models.ChannelRun.find({ + tenantId, + channelId: 'channel', + state: { $in: ['queued', 'executing', 'requires_action', 'delivering', 'blocked'] }, + leaseExpiresAt: { $lte: 305 }, + }) + .sort({ claimCheckedAt: 1, receivedAt: 1, sequence: 1 }) + .limit(25) + .hint('channel_run_fair_claim') + .explain('queryPlanner'); + expect(JSON.stringify(explanation)).toContain('channel_run_fair_claim'); + expect(JSON.stringify(explanation)).not.toMatch(/"stage":"SORT"/); + })); + +it('cleanup ownership supersedes an active worker once and fences stale observations without losing current receipts', () => + scope(async () => { + const run = await ready(); + const inflight = await storage.beginDelivery(tenantId, fence(run), 'part-1', 104); + const cleanup = (await storage.claimAuthorityLost(run, 'cleanup-a', 105, 1000))!; + expect(cleanup.leaseEpoch).toBe(run.leaseEpoch + 1); + expect(cleanup.receipts[0]).toMatchObject({ ...inflight.receipts[0], outcome: 'unknown' }); + expect(await storage.claimAuthorityLost(run, 'stale-cleanup', 106, 1000)).toBeNull(); + expect(await storage.claimAuthorityLost(cleanup, 'cleanup-b', 106, 1000)).toBeNull(); + await expect(storage.beginDelivery(tenantId, fence(run), 'part-2', 106)).rejects.toMatchObject({ + code: 'stale_fence', + }); + await expect(storage.finishAuthorityLost(tenantId, fence(run), 106)).rejects.toMatchObject({ + code: 'stale_fence', + }); + const replacement = (await storage.claimAuthorityLost(cleanup, 'cleanup-b', 1106, 1000))!; + expect(replacement.leaseEpoch).toBe(cleanup.leaseEpoch + 1); + expect(await storage.read(tenantId, 'channel', run.runId)).toMatchObject({ + receipts: cleanup.receipts, + claimCheckedAt: run.claimCheckedAt, + }); + await storage.finishAuthorityLost(tenantId, fence(replacement), 1107); + })); + +it('authority cleanup releases FIFO without advancing history or erasing ambiguous delivery evidence', () => + scope(async () => { + const run = await ready(); + await storage.beginDelivery(tenantId, fence(run), 'part-1', 104); + await expect(storage.finishAuthorityLost(tenantId, fence(run), 105)).rejects.toMatchObject({ + code: 'invalid_transition', + }); + const lost = await storage.markAuthorityLost(tenantId, fence(run), 105); + expect(lost.receipts[0].outcome).toBe('unknown'); + expect(lost.errorCode).toBe('channel_authority_changed'); + await storage.finishAuthorityLost(tenantId, fence(run), 106); + expect(await storage.read(tenantId, 'channel', run.runId)).toMatchObject({ + state: 'cancelled', + receipts: lost.receipts, + }); + expect(await storage.findDeliveredReply(run, 'C1:reply')).toBeNull(); + const next = await storage.admit(enrollment('next'), 107); + const claimed = (await storage.claim(tenantId, 'channel', 'next-worker', 108, 1000))!; + expect(claimed.runId).toBe(next.run.runId); + const prepared = await storage.prepareExecution(tenantId, fence(claimed), 109); + expect(prepared.execution?.parentMessageId).toBe('00000000-0000-0000-0000-000000000000'); + await expect(storage.markAuthorityLost(tenantId, fence(run), 110)).rejects.toMatchObject({ + code: 'stale_fence', + }); + })); diff --git a/packages/api/src/channels/runs.ts b/packages/api/src/channels/runs.ts new file mode 100644 index 0000000000..f416b461e2 --- /dev/null +++ b/packages/api/src/channels/runs.ts @@ -0,0 +1,1033 @@ +import { z } from 'zod'; +import { ChannelRunConflict } from '@librechat/data-schemas'; +import { responseReviewDecisionSchema } from 'librechat-data-provider'; +import type { + ChannelRunPrincipal, + ChannelRunAdmission, + ChannelRunView, + ChannelRunSnapshot, + ChannelRunSummary, + ResponseReviewView, +} from 'librechat-data-provider'; +import type { + AgentChannelRecord, + ChannelRunRecord, + ChannelRunStorage, + ResponseOwner, +} from '@librechat/data-schemas'; +import type { Request, Response, RequestHandler } from 'express'; +import type { ResponseDecisionSubmission } from '~/agents/responses/decision'; +import type { createChannelArtifactService } from './artifacts'; +import type { ChannelServiceGuard } from './types'; +import { ChannelExecutionError } from './execution'; +import { ChannelArtifactError } from './artifacts'; + +export type { ChannelServiceGuard } from './types'; + +export interface ChannelRunDependencies { + interactive?: { + readInteractiveReview: ( + owner: ResponseOwner, + responseId: string, + ) => Promise; + submitInteractiveReview: ResponseDecisionSubmission; + }; + responseCredential?: (run: ChannelRunRecord) => Promise; + storage: ChannelRunStorage; + guard: ChannelServiceGuard; + /** Live tenant membership, agent ACL and explicit channel/link authority, never payload userId. */ + resolvePrincipal: ( + channel: AgentChannelRecord, + actor: ChannelRunAdmission['actor'], + ) => Promise; + advance: ( + run: ChannelRunRecord, + ) => Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }>; + retireForAuthorityLoss: (run: ChannelRunRecord) => Promise<'stopped' | 'pending'>; + now?: () => number; + leaseMs?: number; + getLeaseMs?: () => Promise; + publicBaseUrl?: string; + artifacts?: ReturnType; + /** Optional final metadata only; the caller checks live authority on both sides. */ + summarize?: (run: ChannelRunRecord, snapshot: ChannelRunSnapshot) => Promise; +} + +const id = z + .string() + .min(1) + .max(256) + .regex(/^[\w:.-]+$/); +const actorSchema = z.object({ teamId: id, appId: id, externalUserId: id }).strict(); +const fenceSchema = z.object({ runId: id, owner: id, epoch: z.number().int().positive() }).strict(); +const admissionSchema = z + .object({ + eventId: id, + externalChannelId: id, + externalThreadKey: id, + externalMessageKey: id, + actor: actorSchema, + input: z.string().min(1).max(100_000), + }) + .strict(); +const fenceBody = z.object({ fence: fenceSchema }).strict(); +/** Fixed wire safety bound, not a provider's rendering limit. Content remains opaque to Agents. */ +const presentationSchema = z + .object({ + format: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/), + payload: z + .string() + .max(64_000) + .refine((value) => Buffer.byteLength(value, 'utf8') <= 64_000) + .refine((value) => { + try { + const parsed: unknown = JSON.parse(value); + return parsed !== null && typeof parsed === 'object'; + } catch { + return false; + } + }), + }) + .strict(); +const planBody = z + .object({ + fence: fenceSchema, + plan: z + .object({ + revision: id, + rendererVersion: id, + parts: z + .array( + z + .object({ + operationId: id, + kind: z.enum(['message', 'artifact']), + required: z.boolean(), + text: z.string().max(40_000), + presentation: presentationSchema.optional(), + artifactId: id.optional(), + traceUrl: z.string().url().optional(), + }) + .strict() + .refine((part) => !part.presentation || part.kind === 'message') + .refine((part) => !part.traceUrl || (part.kind === 'message' && !part.required)), + ) + .min(1) + .max(100), + }) + .strict(), + }) + .strict(); +const beginBody = z.object({ fence: fenceSchema, operationId: id }).strict(); +const uploadBody = z + .object({ + fence: fenceSchema, + operationId: id, + attempts: z.number().int().positive(), + upload: z.object({ fileId: id, stage: z.enum(['allocated', 'uploaded']) }).strict(), + }) + .strict(); +const progressRevision = z.number().int().positive(); +const progressPlanBody = z + .object({ + fence: fenceSchema, + revision: progressRevision, + rendererVersion: id, + text: z.string().max(4000), + }) + .strict(); +const progressBeginBody = z.object({ fence: fenceSchema, revision: progressRevision }).strict(); +const progressSettleBody = z + .object({ + fence: fenceSchema, + revision: progressRevision, + receipt: z + .object({ + outcome: z.enum(['confirmed', 'retryable', 'unknown', 'permanent_failure']), + attempts: z.number().int().positive(), + externalMessageKey: id.optional(), + retryAt: z.number().int().nonnegative().optional(), + errorCode: id.optional(), + }) + .strict(), + }) + .strict(); +const settleBody = z + .object({ + fence: fenceSchema, + receipt: z + .object({ + operationId: id, + outcome: z.enum(['confirmed', 'retryable', 'unknown', 'permanent_failure']), + attempts: z.number().int().positive(), + externalMessageKey: id.optional(), + externalFileId: id.optional(), + retryAt: z.number().int().nonnegative().optional(), + errorCode: id.optional(), + }) + .strict(), + }) + .strict(); + +const reconcileDeliveryBody = z + .object({ + fence: fenceSchema, + receipt: z + .object({ + operationId: id, + outcome: z.literal('confirmed'), + attempts: z.number().int().positive(), + externalMessageKey: id.optional(), + externalFileId: id.optional(), + }) + .strict(), + }) + .strict(); +const reconcileProgressBody = z + .object({ + fence: fenceSchema, + revision: progressRevision, + receipt: z + .object({ + outcome: z.literal('confirmed'), + attempts: z.number().int().positive(), + externalMessageKey: id, + }) + .strict(), + }) + .strict(); + +function withoutSession(summary: ChannelRunSummary): ChannelRunSummary { + const { session: _previous, ...metrics } = summary; + return metrics; +} +class ChannelDisclosureError extends Error { + constructor(readonly code: 'channel_authority_changed' | 'trace_unavailable') { + super(code); + } +} + +/** Service projection deliberately omits execution credentials and internal principal identifiers. */ +export function channelRunView(run: ChannelRunRecord): ChannelRunView { + return { + runId: run.runId, + channelId: run.channelId, + revision: run.principal.channelRevision, + state: run.state, + admission: run.admission, + sequence: run.sequence, + snapshot: run.snapshot + ? { + ...run.snapshot, + ...(run.snapshot.summary ? { summary: withoutSession(run.snapshot.summary) } : {}), + } + : undefined, + progress: run.progress, + plan: run.plan + ? { ...run.plan, parts: run.plan.parts.filter((part) => !part.traceUrl) } + : undefined, + receipts: run.receipts.filter( + (receipt) => + !run.plan?.parts.find((part) => part.operationId === receipt.operationId)?.traceUrl, + ), + cancellationRequested: run.cancellationRequested, + }; +} + +export type ChannelRunHandlers = Record< + | 'review' + | 'submitReview' + | 'cancel' + | 'stopThread' + | 'admit' + | 'claim' + | 'renew' + | 'yieldLease' + | 'advance' + | 'prepareResponse' + | 'bindResponse' + | 'plan' + | 'planProgress' + | 'beginProgress' + | 'settleProgress' + | 'reconcileProgress' + | 'reconcileDelivery' + | 'begin' + | 'checkpointUpload' + | 'artifact' + | 'settle' + | 'commit', + RequestHandler +>; + +export async function hasChannelRunAuthority( + channel: AgentChannelRecord, + run: ChannelRunRecord, + resolve: ChannelRunDependencies['resolvePrincipal'], +): Promise { + if (run.errorCode === 'channel_authority_changed' || channel.status !== 'active') return false; + const principal = await resolve(channel, run.admission.actor); + return Boolean( + principal && + principal.userId === run.principal.userId && + principal.channelRevision === run.principal.channelRevision && + principal.linkId === run.principal.linkId && + principal.linkRevision === run.principal.linkRevision, + ); +} + +export function createChannelRunHandlers(deps: ChannelRunDependencies): ChannelRunHandlers { + const now = deps.now ?? Date.now; + const getLeaseMs = deps.getLeaseMs ?? (async () => deps.leaseMs ?? 60_000); + async function hasAuthority(channel: AgentChannelRecord, run: ChannelRunRecord) { + return hasChannelRunAuthority(channel, run, deps.resolvePrincipal); + } + async function presentRun( + channel: AgentChannelRecord, + run: ChannelRunRecord, + ): Promise { + const view = channelRunView(run); + let traceUrl: string | undefined; + if (run.snapshot?.summary && deps.summarize) { + try { + traceUrl = (await deps.summarize(run, run.snapshot)).session?.url; + } catch { + /* Diagnostic metadata is optional; frozen business metrics remain available. */ + } + if (traceUrl && view.snapshot?.summary) + view.snapshot.summary = { ...view.snapshot.summary, session: { url: traceUrl } }; + } + if (!(await hasAuthority(channel, run))) + throw new ChannelDisclosureError('channel_authority_changed'); + if (run.plan) { + const parts = run.plan.parts.filter((part) => !part.traceUrl || part.traceUrl === traceUrl); + view.plan = { ...run.plan, parts }; + view.receipts = run.receipts.filter((receipt) => + parts.some((part) => part.operationId === receipt.operationId), + ); + } + return view; + } + async function retireAuthorityLost( + run: ChannelRunRecord, + fence: Parameters[1], + ) { + const stopped = await deps.storage.markAuthorityLost(run.tenantId, fence, now()); + if ((await deps.retireForAuthorityLoss(stopped)) === 'stopped') + await deps.storage.finishAuthorityLost(run.tenantId, fence, now()); + } + const guarded = (handler: Parameters[0]) => + deps.guard(async (req, res, channel) => { + res.set('Cache-Control', 'no-store'); + try { + await handler(req, res, channel); + } catch (error) { + if (error instanceof ChannelDisclosureError) { + res + .status(error.code === 'channel_authority_changed' ? 403 : 409) + .json({ error: error.code }); + return; + } + if ( + error instanceof ChannelRunConflict || + error instanceof ChannelArtifactError || + error instanceof ChannelExecutionError + ) { + res.status(409).json({ error: error.code }); + return; + } + throw error; + } + }); + async function bound( + req: Request, + res: Response, + channel: AgentChannelRecord, + rawFence: unknown, + ) { + const fence = fenceSchema.safeParse(rawFence); + if (!fence.success || fence.data.runId !== req.params.runId) { + res.status(400).json({ error: 'invalid_run_fence' }); + return null; + } + const run = await deps.storage.read( + channel.tenantId ?? '_', + channel.channelId, + fence.data.runId, + ); + if (!run) { + res.status(404).json({ error: 'run_not_found' }); + return null; + } + if (!(await hasAuthority(channel, run))) { + await retireAuthorityLost(run, fence.data); + res.status(403).json({ error: 'channel_authority_changed' }); + return null; + } + return { run, fence: fence.data }; + } + async function actionRun( + req: Request, + res: Response, + channel: AgentChannelRecord, + decision = false, + ) { + const schema = z.object({ actor: actorSchema, revision: id, externalChannelId: id.optional() }); + const parsed = (decision ? schema.extend({ decision: responseReviewDecisionSchema }) : schema) + .strict() + .safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_run_action' }); + return; + } + const { actor, revision } = parsed.data; + const run = await deps.storage.read( + channel.tenantId ?? '_', + channel.channelId, + req.params.runId, + ); + if (!run) { + res.status(404).json({ error: 'run_not_found' }); + return; + } + if ( + actor.teamId !== run.admission.actor.teamId || + actor.appId !== run.admission.actor.appId || + actor.externalUserId !== run.admission.actor.externalUserId || + (parsed.data.externalChannelId != null && + parsed.data.externalChannelId !== run.admission.externalChannelId) || + revision !== run.principal.channelRevision + ) { + res.status(403).json({ error: 'channel_action_forbidden' }); + return; + } + const principal = await deps.resolvePrincipal(channel, actor); + if ( + !principal || + principal.userId !== run.principal.userId || + principal.channelRevision !== revision || + principal.linkId !== run.principal.linkId || + principal.linkRevision !== run.principal.linkRevision + ) { + res.status(403).json({ error: 'channel_authority_changed' }); + return; + } + return run; + } + return { + stopThread: guarded(async (req, res, channel) => { + const parsed = z + .object({ + actor: actorSchema, + externalThreadKey: id, + externalChannelId: id, + eventAt: z.number().int().positive().max(now()), + }) + .strict() + .safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_run_action' }); + return; + } + const { actor, externalThreadKey, externalChannelId, eventAt } = parsed.data; + if (actor.teamId !== channel.slack?.teamId || actor.appId !== channel.slack?.appId) { + res.status(403).json({ error: 'channel_action_forbidden' }); + return; + } + const principal = await deps.resolvePrincipal(channel, actor); + if (!principal) { + res.status(403).json({ error: 'channel_authority_changed' }); + return; + } + const scope = { + tenantId: channel.tenantId || '_', + channelId: channel.channelId, + agentId: channel.agentId, + principal, + admission: { actor, externalThreadKey, externalChannelId }, + }; + const run = await deps.storage.readActiveThread(scope, eventAt); + if (!run || !(await hasAuthority(channel, run))) { + res.json({ cancellationRequested: false }); + return; + } + const accepted = await deps.storage.requestCancel(scope, run.runId, { + epoch: run.leaseEpoch, + before: eventAt, + }); + res.status(accepted ? 202 : 200).json({ cancellationRequested: accepted }); + }), + review: guarded(async (req, res, channel) => { + const run = await actionRun(req, res, channel); + if (!run) return; + if ( + !deps.publicBaseUrl || + run.cancellationRequested || + run.state !== 'requires_action' || + !run.execution || + (run.execution.protocol === 'responses' && !run.execution.responseId) || + !run.snapshot?.pendingAction || + run.snapshot.pendingAction.expiresAt <= now() + ) { + res.status(409).json({ error: 'run_action_unavailable' }); + return; + } + const url = new URL( + run.execution.protocol === 'responses' + ? `responses/${encodeURIComponent(run.execution.responseId!)}/review` + : `c/${encodeURIComponent(run.execution.conversationId)}`, + `${deps.publicBaseUrl.replace(/\/+$/, '')}/`, + ); + const view = + run.execution.protocol === 'responses' && + deps.interactive && + req.body.externalChannelId === run.admission.externalChannelId + ? await deps.interactive.readInteractiveReview( + { + userId: run.principal.userId, + tenantId: run.tenantId === '_' ? undefined : run.tenantId, + }, + run.execution.responseId!, + ) + : null; + if (!(await hasAuthority(channel, run))) { + res.status(403).json({ error: 'channel_authority_changed' }); + return; + } + const pending = + view?.canSubmitDecision && + view.pendingAction?.actionId === run.snapshot.pendingAction.actionId + ? view.pendingAction + : undefined; + res.json({ + url: url.toString(), + actionId: run.snapshot.pendingAction.actionId, + ...(pending ? { pendingAction: pending } : {}), + ...(pending && view?.pendingDecision ? { pendingDecision: view.pendingDecision } : {}), + }); + }), + submitReview: guarded(async (req, res, channel) => { + const run = await actionRun(req, res, channel, true); + if (!run) return; + if ( + !deps.interactive || + run.execution?.protocol !== 'responses' || + !run.execution.responseId || + run.cancellationRequested || + !req.body.externalChannelId + ) { + res.status(409).json({ error: 'run_action_unavailable' }); + return; + } + // The caller cannot select a principal or Response. Current run authority derives both. + const result = await deps.interactive.submitInteractiveReview( + { userId: run.principal.userId, tenantId: run.tenantId === '_' ? undefined : run.tenantId }, + run.execution.responseId, + req.body.decision, + ); + if (!result.ok) { + res.status(result.status).json({ error: result.code }); + return; + } + res.status(202).json(result.receipt); + }), + cancel: guarded(async (req, res, channel) => { + const run = await actionRun(req, res, channel); + if (!run) return; + const accepted = await deps.storage.requestCancel(run, run.runId); + res + .status(accepted ? 202 : 409) + .json(accepted ? { cancellationRequested: true } : { error: 'run_not_cancellable' }); + }), + admit: guarded(async (req, res, channel) => { + const parsed = admissionSchema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_channel_event' }); + return; + } + const admission = parsed.data; + if ( + admission.actor.teamId !== channel.slack?.teamId || + admission.actor.appId !== channel.slack?.appId + ) { + res.status(403).json({ error: 'channel_actor_mismatch' }); + return; + } + const principal = await deps.resolvePrincipal(channel, admission.actor); + if (!principal) { + res.status(403).json({ error: 'channel_link_required' }); + return; + } + const result = await deps.storage.admit( + { + tenantId: channel.tenantId ?? '_', + channelId: channel.channelId, + agentId: channel.agentId, + principal, + admission, + }, + now(), + ); + res + .status(result.duplicate ? 200 : 201) + .json({ runId: result.run.runId, duplicate: result.duplicate }); + }), + claim: guarded(async (req, res, channel) => { + const parsed = z.object({ owner: id }).strict().safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_run_claim' }); + return; + } + const run = await deps.storage.claim( + channel.tenantId ?? '_', + channel.channelId, + parsed.data.owner, + now(), + await getLeaseMs(), + ); + if (!run) { + res.status(204).end(); + return; + } + if (!(await hasAuthority(channel, run))) { + await retireAuthorityLost(run, { + runId: run.runId, + owner: run.leaseOwner!, + epoch: run.leaseEpoch, + }); + res.status(204).end(); + return; + } + res.json({ + run: await presentRun(channel, run), + fence: { runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch }, + leaseExpiresAt: run.leaseExpiresAt, + }); + }), + renew: guarded(async (req, res, channel) => { + const parsed = fenceBody.safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding) return; + const at = now(); + const leaseMs = await getLeaseMs(); + if (!(await deps.storage.renew(channel.tenantId ?? '_', binding.fence, at, leaseMs))) { + res.status(409).json({ error: 'stale_fence' }); + return; + } + res.json({ fence: binding.fence, leaseExpiresAt: at + leaseMs }); + }), + yieldLease: guarded(async (req, res, channel) => { + const parsed = fenceBody.safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding) return; + if ( + !(await deps.storage.yieldLease( + channel.tenantId ?? '_', + channel.channelId, + binding.fence, + now(), + )) + ) { + res.status(409).json({ error: 'run_yield_unavailable' }); + return; + } + res.status(204).end(); + }), + prepareResponse: guarded(async (req, res, channel) => { + const parsed = fenceBody.safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding) return; + let run = await deps.storage.prepareExecution(channel.tenantId ?? '_', binding.fence, now()); + if (!run.execution || run.execution.deliveryKey || run.execution.enqueueAttemptedAt) { + res.status(409).json({ error: 'execution_protocol_conflict' }); + return; + } + if (run.cancellationRequested && !run.execution.apiKeyId) { + const settled = await deps.storage.finishUnstartedResponseCancellation( + channel.tenantId ?? '_', + binding.fence, + now(), + ); + res.json({ settled: await presentRun(channel, settled) }); + return; + } + if (run.cancellationRequested && !run.execution.responseId) { + const recovered = await deps.storage.recoverResponse( + channel.tenantId ?? '_', + binding.fence, + now(), + ); + if (!recovered) { + res.status(409).json({ error: 'response_cancellation_pending' }); + return; + } + run = recovered; + } + if (!deps.responseCredential) { + res.status(503).json({ error: 'response_credential_unavailable' }); + return; + } + const apiKey = await deps.responseCredential(run); + if (!(await hasAuthority(channel, run))) { + res.status(403).json({ error: 'channel_authority_changed' }); + return; + } + res.json({ + apiKey, + requestKey: `channel:${run.runId}`, + responseId: run.execution!.responseId, + request: { + model: run.agentId, + input: run.admission.input, + background: true, + store: true, + ...(run.execution!.parentMessageId === '00000000-0000-0000-0000-000000000000' + ? {} + : { previous_response_id: run.execution!.parentMessageId }), + }, + }); + }), + bindResponse: guarded(async (req, res, channel) => { + const parsed = z.object({ fence: fenceSchema, responseId: id }).strict().safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding || !parsed.success) return; + const run = await deps.storage.bindResponse( + channel.tenantId ?? '_', + binding.fence, + parsed.data.responseId, + now(), + ); + res.json(await presentRun(channel, run)); + }), + advance: guarded(async (req, res, channel) => { + const parsed = fenceBody.safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding) return; + const { fence } = binding; + let run = binding.run; + if (run.plan || ['delivered', 'failed', 'cancelled'].includes(run.state)) { + res.json(await presentRun(channel, run)); + return; + } + run = await deps.storage.prepareExecution(channel.tenantId ?? '_', fence, now()); + const result = await deps.advance(run); + if (deps.artifacts && result.snapshot.status === 'completed') { + try { + const projection = await deps.artifacts.list(run, result.snapshot); + result.snapshot.artifacts = projection.artifacts; + result.snapshot.artifactsUnavailable = projection.unavailable; + } catch { + result.snapshot.artifactsUnavailable = true; + } + } + if (!(await hasAuthority(channel, run))) { + await retireAuthorityLost(run, fence); + res.status(403).json({ error: 'channel_authority_changed' }); + return; + } + if (run.snapshot?.summary) result.snapshot.summary = run.snapshot.summary; + else if ( + deps.summarize && + ['completed', 'failed', 'cancelled'].includes(result.snapshot.status) + ) { + try { + result.snapshot.summary = await deps.summarize(run, result.snapshot); + } catch { + // Optional accounting/trace metadata must not withhold a valid answer. + delete result.snapshot.summary; + } + if (!(await hasAuthority(channel, run))) { + await retireAuthorityLost(run, fence); + res.status(403).json({ error: 'channel_authority_changed' }); + return; + } + } + run = await deps.storage.saveSnapshot( + channel.tenantId ?? '_', + fence, + result.snapshot, + now(), + result.generationCreatedAt, + ); + res.json(await presentRun(channel, run)); + }), + plan: guarded(async (req, res, channel) => { + const parsed = planBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_delivery_plan' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + if ( + parsed.data.plan.parts.some( + (part) => + part.kind === 'artifact' && + !binding.run.snapshot?.artifacts?.some( + (artifact) => artifact.artifactId === part.artifactId, + ), + ) + ) { + res.status(400).json({ error: 'invalid_artifact_reference' }); + return; + } + const currentView = await presentRun(channel, binding.run); + if ( + parsed.data.plan.parts.some( + (part) => part.traceUrl && part.traceUrl !== currentView.snapshot?.summary?.session?.url, + ) + ) + throw new ChannelDisclosureError('trace_unavailable'); + res.json( + await presentRun( + channel, + await deps.storage.savePlan( + channel.tenantId ?? '_', + binding.fence, + parsed.data.plan, + now(), + ), + ), + ); + }), + planProgress: guarded(async (req, res, channel) => { + const parsed = progressPlanBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_progress_plan' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + const { revision, rendererVersion, text } = parsed.data; + res.json( + await presentRun( + channel, + await deps.storage.planProgress( + channel.tenantId ?? '_', + binding.fence, + { revision, rendererVersion, text }, + now(), + ), + ), + ); + }), + beginProgress: guarded(async (req, res, channel) => { + const parsed = progressBeginBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_progress_operation' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + res.json( + await presentRun( + channel, + await deps.storage.beginProgress( + channel.tenantId ?? '_', + binding.fence, + parsed.data.revision, + now(), + ), + ), + ); + }), + settleProgress: guarded(async (req, res, channel) => { + const parsed = progressSettleBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_progress_receipt' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + const key = parsed.data.receipt.externalMessageKey; + if (key && !key.startsWith(`${binding.run.admission.externalChannelId}:`)) { + res.status(400).json({ error: 'delivery_destination_mismatch' }); + return; + } + res.json( + await presentRun( + channel, + await deps.storage.settleProgress( + channel.tenantId ?? '_', + binding.fence, + { ...parsed.data.receipt, revision: parsed.data.revision }, + now(), + ), + ), + ); + }), + reconcileProgress: guarded(async (req, res, channel) => { + const parsed = reconcileProgressBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_progress_receipt' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + if ( + !parsed.data.receipt.externalMessageKey.startsWith( + `${binding.run.admission.externalChannelId}:`, + ) + ) { + res.status(400).json({ error: 'delivery_destination_mismatch' }); + return; + } + res.json( + await presentRun( + channel, + await deps.storage.reconcileProgress( + channel.tenantId ?? '_', + binding.fence, + { + revision: parsed.data.revision, + attempts: parsed.data.receipt.attempts, + externalMessageKey: parsed.data.receipt.externalMessageKey, + }, + now(), + ), + ), + ); + }), + begin: guarded(async (req, res, channel) => { + const parsed = beginBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_delivery_operation' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + const part = binding.run.plan?.parts.find( + (item) => item.operationId === parsed.data.operationId, + ); + if ( + part?.traceUrl && + !(await presentRun(channel, binding.run)).plan?.parts.some( + (item) => item.operationId === part.operationId, + ) + ) + throw new ChannelDisclosureError('trace_unavailable'); + res.json( + await presentRun( + channel, + await deps.storage.beginDelivery( + channel.tenantId ?? '_', + binding.fence, + parsed.data.operationId, + now(), + ), + ), + ); + }), + checkpointUpload: guarded(async (req, res, channel) => { + const parsed = uploadBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_upload_checkpoint' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + res.json( + await presentRun( + channel, + await deps.storage.checkpointUpload( + channel.tenantId ?? '_', + binding.fence, + parsed.data.operationId, + parsed.data.attempts, + parsed.data.upload, + now(), + ), + ), + ); + }), + artifact: guarded(async (req, res, channel) => { + const parsed = fenceBody.safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding) return; + const leased = (run: ChannelRunRecord) => + run.leaseOwner === binding.fence.owner && + run.leaseEpoch === binding.fence.epoch && + run.leaseExpiresAt > now(); + if (!leased(binding.run)) { + res.status(409).json({ error: 'stale_fence' }); + return; + } + if (!deps.artifacts) { + res.status(404).json({ error: 'artifact_unavailable' }); + return; + } + const content = await deps.artifacts.content(binding.run, req.params.artifactId); + const refreshed = await bound(req, res, channel, binding.fence); + if (!refreshed) return; + if (!leased(refreshed.run)) { + res.status(409).json({ error: 'stale_fence' }); + return; + } + res.set('X-Artifact-Filename', encodeURIComponent(content.descriptor.filename)); + res.set('X-Artifact-Media-Type', content.descriptor.mediaType); + res.set('X-Artifact-SHA256', content.descriptor.sha256); + res.type('application/octet-stream').send(content.bytes); + }), + settle: guarded(async (req, res, channel) => { + const parsed = settleBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_delivery_receipt' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + const key = parsed.data.receipt.externalMessageKey; + if (key && !key.startsWith(`${binding.run.admission.externalChannelId}:`)) { + res.status(400).json({ error: 'delivery_destination_mismatch' }); + return; + } + res.json( + await presentRun( + channel, + await deps.storage.settleDelivery( + channel.tenantId ?? '_', + binding.fence, + parsed.data.receipt, + now(), + ), + ), + ); + }), + reconcileDelivery: guarded(async (req, res, channel) => { + const parsed = reconcileDeliveryBody.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'invalid_delivery_receipt' }); + return; + } + const binding = await bound(req, res, channel, parsed.data.fence); + if (!binding) return; + const key = parsed.data.receipt.externalMessageKey; + if (key && !key.startsWith(`${binding.run.admission.externalChannelId}:`)) { + res.status(400).json({ error: 'delivery_destination_mismatch' }); + return; + } + res.json( + await presentRun( + channel, + await deps.storage.reconcileDelivery( + channel.tenantId ?? '_', + binding.fence, + parsed.data.receipt, + now(), + ), + ), + ); + }), + commit: guarded(async (req, res, channel) => { + const parsed = fenceBody.safeParse(req.body); + const binding = await bound(req, res, channel, parsed.success ? parsed.data.fence : null); + if (!binding) return; + res.json( + await presentRun( + channel, + await deps.storage.commitDelivery(channel.tenantId ?? '_', binding.fence, now()), + ), + ); + }), + }; +} diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index fbbdd6541d..03a1162b76 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -158,12 +158,14 @@ export * from './mcp/approval'; export { createChannelStorageConfigReader } from './channels/config'; export * from './channels/service'; - +export * from './channels/runs'; export * from './channels/authority'; - +export * from './channels/execution'; +export * from './channels/responses'; +export * from './channels/credentials'; export * from './channels/identity'; export * from './channels/identityHttp'; export * from './channels/artifacts'; export * from './channels/artifactStorage'; - +export * from './channels/cleanup'; export * from './channels/summary'; diff --git a/packages/data-provider/src/types/channelRuns.ts b/packages/data-provider/src/types/channelRuns.ts index 4b7a134ff9..7dc416537e 100644 --- a/packages/data-provider/src/types/channelRuns.ts +++ b/packages/data-provider/src/types/channelRuns.ts @@ -111,6 +111,8 @@ export interface ChannelDeliveryPart { text: string; /** Provider-defined, versioned JSON presentation; immutable alongside the text fallback. */ presentation?: { format: string; payload: string }; + /** Optional diagnostic operation; reauthorized independently of immutable answer delivery. */ + traceUrl?: string; /** Opaque server-authorized artifact identity; never an arbitrary URL or disk path. */ artifactId?: string; } diff --git a/packages/data-schemas/src/methods/channelProtocol.ts b/packages/data-schemas/src/methods/channelProtocol.ts index 6bacd5d293..2b24a9abbf 100644 --- a/packages/data-schemas/src/methods/channelProtocol.ts +++ b/packages/data-schemas/src/methods/channelProtocol.ts @@ -187,6 +187,7 @@ export function assertPlan( (part) => !part.operationId || !['message', 'artifact'].includes(part.kind) || + (part.traceUrl != null && (part.required || part.kind !== 'message' || !part.traceUrl)) || (part.kind === 'artifact' && (!part.artifactId || !run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))), diff --git a/packages/data-schemas/src/methods/response.spec.ts b/packages/data-schemas/src/methods/response.spec.ts index 0ee6aa9d74..015636f13d 100644 --- a/packages/data-schemas/src/methods/response.spec.ts +++ b/packages/data-schemas/src/methods/response.spec.ts @@ -2193,3 +2193,23 @@ it('closes and cooperatively cancels private prepared work while retaining redac await eraseResponseRecords(mongoose, owner, ['conversation-one']); expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0); })); + +it.each(['live', 'expired', 'closed'] as const)( + 'returns immutable cleanup binding metadata without payload for %s rows', + (state) => + scoped(async () => { + await storage.reserve(reservation({ sourceKeyId: 'run-key' })); + if (state === 'expired') + await createResponseRetention(mongoose)(owner, 'conversation-one', new Date(0)); + if (state === 'closed') await closeResponseScopes(mongoose, owner, ['conversation-one']); + const identity = await storage.readCleanupIdentity(owner, 'resp_one'); + expect(identity).toMatchObject({ + requestKey: 'request-one', + sourceKeyId: 'run-key', + parentMessageId: 'root', + }); + expect(identity).not.toHaveProperty('requestJSON'); + expect(identity).not.toHaveProperty('responseJSON'); + expect(JSON.stringify(identity)).not.toContain('Hello'); + }), +); diff --git a/packages/data-schemas/src/methods/response.ts b/packages/data-schemas/src/methods/response.ts index 7715cd68f5..0d2abbd499 100644 --- a/packages/data-schemas/src/methods/response.ts +++ b/packages/data-schemas/src/methods/response.ts @@ -52,7 +52,8 @@ export type ResponseMutationResult = | { ok: true; record: StoredResponseRecord } | { ok: false; code: 'response_conflict' }; -/** Internal drain identity only; expired requests, answers and decisions are never returned. */ +/** Internal drain identity only; expired requests, answers and decisions are never returned. + * Stored rows include immutable request binding; older retained job identities may omit it. */ export type ResponseCleanupIdentity = Pick< StoredResponseRecord, | 'userId' @@ -62,13 +63,14 @@ export type ResponseCleanupIdentity = Pick< | 'conversationId' | 'generationCreatedAt' | 'expiredAt' -> & { - /** Cleanup authority only; never permission to read output or execute a provider. */ - retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired'; - checkpointNamespace?: string; - checkpointConfigJSON?: string; - requiresCheckpointCleanup?: boolean; -}; +> & + Partial> & { + /** Cleanup authority only; never permission to read output or execute a provider. */ + retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired'; + checkpointNamespace?: string; + checkpointConfigJSON?: string; + requiresCheckpointCleanup?: boolean; + }; export interface ResponseResumeReservation extends Omit { @@ -851,7 +853,7 @@ export function createResponseStorage( const record = await model .findOne({ ...ownerScope(owner), responseId }) .select( - 'userId tenantId responseId agentId conversationId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON preparedTerminal.checkpointNamespace preparedTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId', + 'userId tenantId responseId agentId conversationId requestKey sourceKeyId parentMessageId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON preparedTerminal.checkpointNamespace preparedTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId', ) .lean(); // Retained data comes only from an internal job/receipt, never request input. @@ -928,6 +930,9 @@ export function createResponseStorage( responseId: id, agentId, conversationId, + ...(source.requestKey == null ? {} : { requestKey: source.requestKey }), + ...(source.sourceKeyId == null ? {} : { sourceKeyId: source.sourceKeyId }), + ...(source.parentMessageId == null ? {} : { parentMessageId: source.parentMessageId }), generationCreatedAt: generationCreatedAt ?? (retirement ? matchingRetained?.generationCreatedAt : undefined), expiredAt,