diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js index 4a50577d31..2f3b9b9b25 100644 --- a/api/server/controllers/agents/__tests__/resume.spec.js +++ b/api/server/controllers/agents/__tests__/resume.spec.js @@ -221,6 +221,26 @@ const { captureCodeExecutionApprovalBinding } = require('@librechat/api'); /** Drain the microtask + immediate queues so the post-ACK continuation settles. */ const flush = () => new Promise((resolve) => setImmediate(resolve)); +/** Provenance commits in the approval claim's CAS, never in a post-ACK job-store write. */ +const expectProvenanceClaimed = (provenance) => { + expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith( + CONVO_ID, + expect.anything(), + expect.objectContaining(provenance), + 1000, + ); + expect(mockJobStore.updateJob).not.toHaveBeenCalledWith( + CONVO_ID, + expect.objectContaining({ userSubmittedPaths: expect.anything() }), + expect.anything(), + ); + expect(mockJobStore.updateJob).not.toHaveBeenCalledWith( + CONVO_ID, + expect.objectContaining({ userSubmittedMessageFieldPaths: expect.anything() }), + expect.anything(), + ); +}; + /** A live, resolvable paused tool-approval job (single tool call `tc1`). */ function makeToolApprovalJob(overrides = {}) { const metaOverrides = overrides.metadata ?? {}; @@ -1284,6 +1304,42 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { expect(mockDeleteAgentCheckpoint).toHaveBeenCalled(); }); + it('aborts a resume the schedule fence rejected after claiming its provenance', async () => { + const job = makeScheduledJob(); + job.metadata.userSubmittedPaths = ['/content/0/steer']; + job.metadata.pendingAction.payload.review_configs = [ + { tool_call_id: 'tc1', allowed_decisions: ['approve', 'edit'] }, + ]; + mockGenerationJobManager.getJob.mockResolvedValue(job); + mockGenerationJobManager.getResumeState.mockResolvedValue({ + aggregatedContent: [makeToolCallContent()], + }); + mockFinalizeScheduleResumeClaim.mockResolvedValue(false); + + const res = await post( + approveBody({ + decisions: [{ tool_call_id: 'tc1', decision: 'edit', editedArguments: { q: 'edit' } }], + }), + ); + + expect(res.status).toBe(409); + expect(res.body).toMatchObject({ code: 'SCHEDULE_NO_LONGER_ACTIVE' }); + expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith( + CONVO_ID, + ACTION_ID, + expect.objectContaining({ + userSubmittedPaths: expect.arrayContaining(['/content/0/tool_call/args']), + }), + 1000, + ); + expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(CONVO_ID, { + expectedCreatedAt: 1000, + awaitProviderDrain: true, + }); + expect(mockJobStore.updateJob).not.toHaveBeenCalled(); + expect(mockInitializeClient).not.toHaveBeenCalled(); + }); + it('refuses to settle the stale resume handoff on an unconfirmed stop', async () => { mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob()); mockFinalizeScheduleResumeClaim.mockResolvedValue(false); @@ -3970,15 +4026,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { }), 1000, ); - expect(mockJobStore.updateJob).toHaveBeenCalledWith( - CONVO_ID, - { - userSubmittedMessageFieldPaths: [ - { path: '/content/1/tool_call/output', field: 'answer' }, - ], - }, - 1000, - ); + expectProvenanceClaimed({ + userSubmittedMessageFieldPaths: [{ path: '/content/1/tool_call/output', field: 'answer' }], + }); expect(mockSaveMessage).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ @@ -4085,11 +4135,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { 1000, ); const exactProvenance = [{ path: '/content/0/tool_call/output', field: 'answer' }]; - expect(mockJobStore.updateJob).toHaveBeenCalledWith( - CONVO_ID, - { userSubmittedMessageFieldPaths: exactProvenance }, - 1000, - ); + expectProvenanceClaimed({ userSubmittedMessageFieldPaths: exactProvenance }); expect(mockSaveMessage).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ userSubmittedMessageFieldPaths: exactProvenance }), @@ -4142,11 +4188,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { await flush(); const exactProvenance = [{ path: '/content/0/tool_call/output', field: 'answer' }]; - expect(mockJobStore.updateJob).toHaveBeenCalledWith( - CONVO_ID, - { userSubmittedMessageFieldPaths: exactProvenance }, - 1000, - ); + expectProvenanceClaimed({ userSubmittedMessageFieldPaths: exactProvenance }); expect(mockSaveMessage).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ userSubmittedMessageFieldPaths: exactProvenance }), @@ -4206,7 +4248,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { userSubmittedMessageFieldPaths: [{ path: expectedPath, field: expectedField }], } : { userSubmittedPaths: [expectedPath] }; - expect(mockJobStore.updateJob).toHaveBeenCalledWith(CONVO_ID, expectedProvenance, 1000); + expectProvenanceClaimed(expectedProvenance); expect(mockSaveMessage).toHaveBeenCalledWith( expect.anything(), expect.objectContaining(expectedProvenance), @@ -4236,6 +4278,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { await settled; await flush(); + expect(mockGenerationJobManager.approvals.resolve.mock.calls[0][2]).not.toHaveProperty( + 'userSubmittedPaths', + ); expect(mockJobStore.updateJob).not.toHaveBeenCalledWith( CONVO_ID, expect.objectContaining({ userSubmittedPaths: expect.anything() }), @@ -4594,15 +4639,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { expect(mockSaveMessage).not.toHaveBeenCalled(); expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled(); expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled(); - expect(mockJobStore.updateJob).toHaveBeenCalledWith( - CONVO_ID, - { - userSubmittedMessageFieldPaths: [ - { path: '/content/0/tool_call/output', field: 'decision_response' }, - ], - }, - 1000, - ); + expectProvenanceClaimed({ + userSubmittedMessageFieldPaths: [ + { path: '/content/0/tool_call/output', field: 'decision_response' }, + ], + }); expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID); }); diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js index c7ec361c7e..77511c55de 100644 --- a/api/server/controllers/agents/resume.js +++ b/api/server/controllers/agents/resume.js @@ -1557,6 +1557,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) providerExecutionId, providerDrained: true, ...(resolvedAskUserQuestion && { resolvedAskUserQuestions }), + ...(userSubmittedPaths.length > 0 && { userSubmittedPaths }), + ...(userSubmittedMessageFieldPaths.length > 0 && { userSubmittedMessageFieldPaths }), }, job.createdAt, ); @@ -1932,18 +1934,6 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) if (userSubmittedMessageFieldPaths.length > 0) { job.metadata.userSubmittedMessageFieldPaths = userSubmittedMessageFieldPaths; } - if (userSubmittedPaths.length > 0 || userSubmittedMessageFieldPaths.length > 0) { - await GenerationJobManager.getJobStore().updateJob( - streamId, - { - ...(userSubmittedPaths.length > 0 && { userSubmittedPaths }), - ...(userSubmittedMessageFieldPaths.length > 0 && { - userSubmittedMessageFieldPaths, - }), - }, - job.createdAt, - ); - } const mcpRequestBody = job.metadata.mcpRequestBody ?? diff --git a/e2e/specs/mock/approvals.helpers.ts b/e2e/specs/mock/approvals.helpers.ts index 885345f6b3..7ab9961117 100644 --- a/e2e/specs/mock/approvals.helpers.ts +++ b/e2e/specs/mock/approvals.helpers.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { expect } from '@playwright/test'; -import type { Page, Request } from '@playwright/test'; +import type { Locator, Page, Request } from '@playwright/test'; import type { AgentDetail } from './agents.helpers'; import { openAgentBuilder, uniqueAgentName } from './agents.helpers'; import { @@ -11,6 +11,7 @@ import { getAccessToken, messagesView, requestJson, + sendMessage, } from './helpers'; const MCP_SERVER_NAME = 'e2e-memory'; @@ -110,18 +111,17 @@ async function waitForApprovalTool(page: Page) { ).toEqual(expect.arrayContaining([expect.objectContaining({ pluginKey: APPROVAL_TOOL_ID })])); } -export async function createAndSelectApprovalAgent(page: Page): Promise { +export async function createApprovalAgent(page: Page): Promise { await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); await waitForApprovalTool(page); const token = await getAccessToken(page); - const agentName = uniqueAgentName('E2E Tool Approval Agent'); - const agent = await requestJson(page, { + return requestJson(page, { path: '/api/agents', token, method: 'POST', body: { - name: agentName, + name: uniqueAgentName('E2E Tool Approval Agent'), description: DESCRIPTION, instructions: 'Use the requested approval probe tools and report their results.', provider: MOCK_ENDPOINTS[0].label, @@ -129,11 +129,96 @@ export async function createAndSelectApprovalAgent(page: Page): Promise tools: [MCP_SERVER_TOOL_ID, APPROVAL_TOOL_ID], }, }); +} +export async function createAndSelectApprovalAgent(page: Page): Promise { + const agent = await createApprovalAgent(page); const form = await openAgentBuilder(page); await form.getByRole('combobox', { name: 'Agent', exact: true }).click(); - await page.getByRole('option', { name: agentName }).click(); - await expect(form.getByLabel('Agent name')).toHaveValue(agentName); + await page.getByRole('option', { name: agent.name }).click(); + await expect(form.getByLabel('Agent name')).toHaveValue(agent.name); await form.getByRole('button', { name: 'Select Agent' }).click(); return agent.id; } + +export async function startApproval( + page: Page, + label: string, + marker = APPROVAL_PROMPT_MARKER, + expectedReason = APPROVAL_REASON, +): Promise { + const response = await sendMessage(page, `${marker}${label}`); + expect(response.ok()).toBeTruthy(); + await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 }); + const card = approvalCards(page).first(); + await expect(card).toBeVisible({ timeout: 30000 }); + await expect(card).toContainText(expectedReason); + /** + * The primary composer review opens automatically above the historical + * timeline card. Verify that entry point, then collapse it so these tests + * can keep exercising the timeline fallback without an overlay intercepting + * its controls. The native BYOM acceptance spec submits through the composer. + */ + await collapseComposerApproval(page); + return card; +} + +export async function submitAndCapture(page: Page, submit: Locator) { + const [request, response] = await Promise.all([ + page.waitForRequest(isResumeRequest), + page.waitForResponse( + (candidate) => isResumeRequest(candidate.request()) && candidate.status() === 200, + ), + submit.click(), + ]); + return { + body: request.postDataJSON() as ApprovalResumeBody, + response, + }; +} + +export async function expectCompletedApprovalToolOutput( + page: Page, + toolCallId: string, + output: string, +) { + const view = messagesView(page); + const groupToggle = view.getByRole('button', { name: /^Ran \d+ actions/ }).last(); + const toolCall = view.locator(`[data-testid="tool-call"][data-tool-call-id="${toolCallId}"]`); + + // On reload, the conversation arrives asynchronously and multi-tool groups + // start collapsed. Wait for either the target card or its group before + // deciding whether expansion is necessary. + await expect(toolCall.or(groupToggle).first()).toBeVisible({ timeout: 30000 }); + // The final model turn is the quiescence barrier: all parallel tool work + // has settled before invocation-count assertions inspect the audit. It is + // also the fence the expansions below need, because the streamed response + // carries a placeholder id that the saved message replaces, remounting + // every card in the turn and closing whatever this helper had opened. + await expect(view.getByText(/^E2E approval outcomes:/).last()).toBeVisible({ timeout: 30000 }); + + const toggle = toolCall.getByRole('button', { name: /Ran approval_probe/ }); + // Scope exact output to its stable call id. This catches both a dropped + // completion and an output accidentally attached to a sibling tool card. + const toolOutput = view + .locator(`[data-tool-call-output-id="${toolCallId}"]`) + .getByText(output, { exact: true }); + + // Re-open on every attempt rather than expanding once: a card that a late + // remount closes underneath would otherwise leave the assertion waiting on + // a body that nothing is going to mount again. + await expect(async () => { + if (!(await toolCall.isVisible())) { + const hasGroup = (await groupToggle.count()) > 0; + if (hasGroup && (await groupToggle.getAttribute('aria-expanded')) !== 'true') { + await groupToggle.click(); + } + } + await expect(toolCall).toBeVisible({ timeout: 5000 }); + await expect(toggle).toBeVisible({ timeout: 5000 }); + if ((await toggle.getAttribute('aria-expanded')) !== 'true') { + await toggle.click(); + } + await expect(toolOutput).toBeVisible({ timeout: 5000 }); + }).toPass({ timeout: 30000 }); +} diff --git a/e2e/specs/mock/scenarios/approval-provenance.spec.ts b/e2e/specs/mock/scenarios/approval-provenance.spec.ts new file mode 100644 index 0000000000..d7a6d6a805 --- /dev/null +++ b/e2e/specs/mock/scenarios/approval-provenance.spec.ts @@ -0,0 +1,74 @@ +import { expect, test } from '@playwright/test'; +import { + uniqueLabel, + startApproval, + submitAndCapture, + clearApprovalInvocations, + createApprovalAgent, + expectApprovalInvocationCount, + expectCompletedApprovalToolOutput, +} from '../approvals.helpers'; +import { cleanupAgent } from '../agents.helpers'; +import { NEW_CHAT_PATH, getAccessToken, requestJson } from '../helpers'; + +type SavedMessage = { isCreatedByUser?: boolean; userSubmittedPaths?: string[] }; + +test.describe('Tool approval provenance', () => { + test('an edited approval resumes and its saved reply marks the edit as user-submitted @scenario:an-edited-approval-keeps-its-user-submitted-provenance', async ({ + page, + }) => { + test.setTimeout(120000); + const label = uniqueLabel(); + const toolCallId = `call_e2e_approval_${label}`; + const originalValue = `original-${label}`; + const editedValue = `edited-${label}`; + let agentId: string | undefined; + clearApprovalInvocations(originalValue, editedValue); + + try { + agentId = (await createApprovalAgent(page)).id; + await page.goto(`${NEW_CHAT_PATH}?agent_id=${encodeURIComponent(agentId)}`, { + timeout: 10000, + }); + const card = await startApproval(page, label); + + await card.getByRole('button', { name: 'Edit' }).click(); + await card + .getByRole('textbox', { name: 'Edit' }) + .fill(JSON.stringify({ value: editedValue })); + const { body, response } = await submitAndCapture( + page, + card.getByRole('button', { name: 'Submit' }), + ); + expect((await response.json()).status).toBe('resuming'); + const conversationId = body.conversationId ?? ''; + expect(conversationId).not.toBe(''); + + await expectCompletedApprovalToolOutput( + page, + toolCallId, + `E2E approval probe executed: ${editedValue}`, + ); + await expectApprovalInvocationCount(editedValue, 1); + + const token = await getAccessToken(page); + await expect + .poll( + async () => { + const messages = await requestJson(page, { + path: `/api/messages/${encodeURIComponent(conversationId)}`, + token, + }); + return messages.find((message) => !message.isCreatedByUser)?.userSubmittedPaths ?? []; + }, + { timeout: 30000 }, + ) + .toEqual( + expect.arrayContaining([expect.stringMatching(/^\/content\/\d+\/tool_call\/args$/)]), + ); + } finally { + clearApprovalInvocations(originalValue, editedValue); + await cleanupAgent(page, agentId); + } + }); +}); diff --git a/e2e/specs/mock/tool-approvals.spec.ts b/e2e/specs/mock/tool-approvals.spec.ts index 66feeee12c..b8f7a1b8a7 100644 --- a/e2e/specs/mock/tool-approvals.spec.ts +++ b/e2e/specs/mock/tool-approvals.spec.ts @@ -1,11 +1,10 @@ import { expect, test } from '@playwright/test'; -import type { Locator, Page, Route } from '@playwright/test'; -import type { ApprovalResumeBody, ApprovalResumeResponse } from './approvals.helpers'; +import type { Route } from '@playwright/test'; +import type { ApprovalResumeResponse } from './approvals.helpers'; import { APPROVAL_ERROR, APPROVAL_REASON, APPROVAL_EXPIRED, - APPROVAL_PROMPT_MARKER, BATCH_APPROVAL_PROMPT_MARKER, REWRITTEN_APPROVAL_PROMPT_MARKER, RESTRICTED_APPROVAL_PROMPT_MARKER, @@ -15,90 +14,15 @@ import { isResumeRequest, collapseComposerApproval, clearApprovalInvocations, + startApproval, + submitAndCapture, createAndSelectApprovalAgent, expectApprovalInvocationCount, + expectCompletedApprovalToolOutput, } from './approvals.helpers'; -import { NEW_CHAT_PATH, getAccessToken, messagesView, requestJson, sendMessage } from './helpers'; +import { NEW_CHAT_PATH, getAccessToken, messagesView, requestJson } from './helpers'; import { cleanupAgent } from './agents.helpers'; -async function startApproval( - page: Page, - label: string, - marker = APPROVAL_PROMPT_MARKER, - expectedReason = APPROVAL_REASON, -): Promise { - const response = await sendMessage(page, `${marker}${label}`); - expect(response.ok()).toBeTruthy(); - await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 }); - const card = approvalCards(page).first(); - await expect(card).toBeVisible({ timeout: 30000 }); - await expect(card).toContainText(expectedReason); - /** - * The primary composer review opens automatically above the historical - * timeline card. Verify that entry point, then collapse it so these tests - * can keep exercising the timeline fallback without an overlay intercepting - * its controls. The native BYOM acceptance spec submits through the composer. - */ - await collapseComposerApproval(page); - return card; -} - -async function submitAndCapture(page: Page, submit: Locator) { - const [request, response] = await Promise.all([ - page.waitForRequest(isResumeRequest), - page.waitForResponse( - (candidate) => isResumeRequest(candidate.request()) && candidate.status() === 200, - ), - submit.click(), - ]); - return { - body: request.postDataJSON() as ApprovalResumeBody, - response, - }; -} - -async function expectCompletedApprovalToolOutput(page: Page, toolCallId: string, output: string) { - const view = messagesView(page); - const groupToggle = view.getByRole('button', { name: /^Ran \d+ actions/ }).last(); - const toolCall = view.locator(`[data-testid="tool-call"][data-tool-call-id="${toolCallId}"]`); - - // On reload, the conversation arrives asynchronously and multi-tool groups - // start collapsed. Wait for either the target card or its group before - // deciding whether expansion is necessary. - await expect(toolCall.or(groupToggle).first()).toBeVisible({ timeout: 30000 }); - // The final model turn is the quiescence barrier: all parallel tool work - // has settled before invocation-count assertions inspect the audit. It is - // also the fence the expansions below need, because the streamed response - // carries a placeholder id that the saved message replaces, remounting - // every card in the turn and closing whatever this helper had opened. - await expect(view.getByText(/^E2E approval outcomes:/).last()).toBeVisible({ timeout: 30000 }); - - const toggle = toolCall.getByRole('button', { name: /Ran approval_probe/ }); - // Scope exact output to its stable call id. This catches both a dropped - // completion and an output accidentally attached to a sibling tool card. - const toolOutput = view - .locator(`[data-tool-call-output-id="${toolCallId}"]`) - .getByText(output, { exact: true }); - - // Re-open on every attempt rather than expanding once: a card that a late - // remount closes underneath would otherwise leave the assertion waiting on - // a body that nothing is going to mount again. - await expect(async () => { - if (!(await toolCall.isVisible())) { - const hasGroup = (await groupToggle.count()) > 0; - if (hasGroup && (await groupToggle.getAttribute('aria-expanded')) !== 'true') { - await groupToggle.click(); - } - } - await expect(toolCall).toBeVisible({ timeout: 5000 }); - await expect(toggle).toBeVisible({ timeout: 5000 }); - if ((await toggle.getAttribute('aria-expanded')) !== 'true') { - await toggle.click(); - } - await expect(toolOutput).toBeVisible({ timeout: 5000 }); - }).toPass({ timeout: 30000 }); -} - test.describe('tool approvals', () => { test('approves a paused tool with its original arguments', async ({ page }) => { test.setTimeout(120000); diff --git a/packages/api/src/stream/ApprovalLifecycle.ts b/packages/api/src/stream/ApprovalLifecycle.ts index e95aefde01..9b15fd2be0 100644 --- a/packages/api/src/stream/ApprovalLifecycle.ts +++ b/packages/api/src/stream/ApprovalLifecycle.ts @@ -400,6 +400,10 @@ export class ApprovalLifecycle { const { steerQuotesCapable, ...ownerPatch } = resumePatch ?? {}; const boundPatch = { ...ownerPatch, + preResumeProvenance: { + userSubmittedPaths: job.userSubmittedPaths ?? [], + userSubmittedMessageFieldPaths: job.userSubmittedMessageFieldPaths ?? [], + }, ...(steerQuotesCapable === true && typeof ownerPatch.providerExecutionId === 'string' && { steerQuotesExecutionId: ownerPatch.providerExecutionId, diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index 3d28faabdc..71f88ae52d 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -82,6 +82,7 @@ import { synthesizeReasoningLabelGapEvents } from '~/agents/reasoningLabels'; import { InMemoryEventTransport } from './implementations/InMemoryEventTransport'; import { InMemoryJobStore } from './implementations/InMemoryJobStore'; import { attachAskUserQuestionAnswers, normalizeResumeRunStepIndices } from '~/agents/hitl/resume'; +import { ASK_USER_QUESTION_TOOL_NAME } from '~/agents/hitl/askUserQuestionTool'; import { emitChunkWithReceipt } from './internal/chunkPublication'; import { resolveCoalesceWindowMs } from './internal/coalescing'; import { @@ -369,6 +370,93 @@ function getSteerUserSubmittedPaths(content: readonly TMessageContentParts[]): s return paths; } +/** Rewrites `/content/N/...` paths recorded against the unfiltered content onto + * the filtered abort content, dropping paths whose part was filtered out. A + * path past the end of the content names no part and is left as recorded. */ +function remapContentPath( + path: string, + contentLength: number, + indexMap: ReadonlyMap, +): string | null { + const match = /^\/content\/(\d+)(\/.*)?$/.exec(path); + if (match == null || Number(match[1]) >= contentLength) { + return path; + } + const index = indexMap.get(Number(match[1])); + return index == null ? null : `/content/${index}${match[2] ?? ''}`; +} + +/** A tool-call path the latest approval claim added is user-authored only once + * its decision reached this content: the resumed call completed (a string + * output, which may be empty for a void tool) or the abort route stamped the + * answer. An unanswered `ask_user_question` already holds an empty output, so + * only a non-empty one counts as its answer. Other claimed paths (steers already + * in the seed content) need no decision to apply and are kept. Paths are checked + * against the unfiltered content they were recorded on, then remapped onto the + * filtered abort content that the final event and the persisted row carry. */ +function getPublishedProvenance( + jobData: SerializableJobData, + content: readonly unknown[], + abortContent: readonly unknown[], +): Pick { + const claimedPaths = jobData.userSubmittedPaths ?? []; + const claimedFieldPaths = jobData.userSubmittedMessageFieldPaths ?? []; + const preResume = jobData.preResumeProvenance; + const prePaths = new Set(preResume?.userSubmittedPaths ?? []); + const preFieldPaths = new Set( + (preResume?.userSubmittedMessageFieldPaths ?? []).map(({ path, field }) => `${field}:${path}`), + ); + const isAppliedPath = (path: string): boolean => { + if (preResume == null) { + return true; + } + const match = /^\/content\/(\d+)\/tool_call\//.exec(path); + if (match == null) { + return true; + } + const part = content[Number(match[1])] as TMessageContentParts | undefined; + if (part?.type !== 'tool_call') { + return false; + } + const toolCall = part.tool_call as { name?: unknown; output?: unknown } | undefined; + const output = toolCall?.output; + if (typeof output !== 'string') { + return false; + } + return output.length > 0 || toolCall?.name !== ASK_USER_QUESTION_TOOL_NAME; + }; + /** The filter keeps part references in order, so a forward scan maps them. */ + const indexMap = new Map(); + for (let index = 0, filtered = 0; index < content.length; index++) { + if (filtered < abortContent.length && content[index] === abortContent[filtered]) { + indexMap.set(index, filtered++); + } + } + const userSubmittedPaths: string[] = []; + for (const path of claimedPaths) { + const remapped = + prePaths.has(path) || isAppliedPath(path) + ? remapContentPath(path, content.length, indexMap) + : null; + if (remapped != null) { + userSubmittedPaths.push(remapped); + } + } + const userSubmittedMessageFieldPaths: NonNullable< + SerializableJobData['userSubmittedMessageFieldPaths'] + > = []; + for (const entry of claimedFieldPaths) { + const remapped = + preFieldPaths.has(`${entry.field}:${entry.path}`) || isAppliedPath(entry.path) + ? remapContentPath(entry.path, content.length, indexMap) + : null; + if (remapped != null) { + userSubmittedMessageFieldPaths.push({ ...entry, path: remapped }); + } + } + return { userSubmittedPaths, userSubmittedMessageFieldPaths }; +} + function getToolCallName(toolCall: unknown): unknown { return toolCall != null && typeof toolCall === 'object' && 'name' in toolCall ? toolCall.name @@ -4816,6 +4904,12 @@ class GenerationJobManagerClass { /** Final event for abort */ const userMessageId = jobData.userMessage?.messageId; + /** The final event and the persisted row (`beforePublish` reads + * `jobData`) must label the same content, so both take this selection. */ + jobData = { + ...jobData, + ...getPublishedProvenance(jobData, content, abortContent), + }; const userSubmittedPaths = [ ...new Set([ ...(jobData.userSubmittedPaths ?? []), diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts index 30068cea15..639c93ee58 100644 --- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts +++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts @@ -213,6 +213,51 @@ describe('RedisJobStore Integration Tests', () => { await store.destroy(); }); + test('persists user-submitted provenance patched by a resume transition', async () => { + if (!ioredisClient) { + return; + } + + const { RedisJobStore } = await import('../implementations/RedisJobStore'); + const store = new RedisJobStore(ioredisClient); + await store.initialize(); + + const streamId = `test-transition-provenance-${Date.now()}`; + const created = await store.createJob(streamId, 'provenance-user', streamId); + await store.transitionStatus(streamId, { from: 'running', to: 'requires_action' }); + const userSubmittedMessageFieldPaths = [ + { path: '/content/0/tool_call/output', field: 'answer' as const }, + ]; + + await expect( + store.transitionStatus(streamId, { + from: 'requires_action', + to: 'running', + patch: { + userSubmittedPaths: ['/content/0/tool_call/args'], + userSubmittedMessageFieldPaths, + preResumeProvenance: { + userSubmittedPaths: ['/content/0/steer'], + userSubmittedMessageFieldPaths: [], + }, + }, + expectCreatedAt: created.createdAt, + }), + ).resolves.toBe(true); + + await expect(store.getJob(streamId)).resolves.toMatchObject({ + status: 'running', + userSubmittedPaths: ['/content/0/tool_call/args'], + userSubmittedMessageFieldPaths, + preResumeProvenance: { + userSubmittedPaths: ['/content/0/steer'], + userSubmittedMessageFieldPaths: [], + }, + }); + + await store.destroy(); + }); + test('returns the exact predecessor captured by the atomic replacement script', async () => { if (!ioredisClient) { return; diff --git a/packages/api/src/stream/__tests__/pendingAction.spec.ts b/packages/api/src/stream/__tests__/pendingAction.spec.ts index 33f2be2638..d1bb5df09d 100644 --- a/packages/api/src/stream/__tests__/pendingAction.spec.ts +++ b/packages/api/src/stream/__tests__/pendingAction.spec.ts @@ -829,6 +829,38 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', () ); }); + test('persists user-submitted provenance in the same CAS that resumes the job', async () => { + const streamId = 'stream-resolve-provenance'; + const job = await manager.createJob(streamId, 'user-1'); + const action = buildAction(streamId); + await manager.approvals.pause(streamId, action); + const userSubmittedMessageFieldPaths = [ + { path: '/content/0/tool_call/output', field: 'decision_response' as const }, + ]; + + expect( + await manager.approvals.resolve( + streamId, + action.actionId, + { + userSubmittedPaths: ['/content/0/tool_call/args'], + userSubmittedMessageFieldPaths, + }, + job.createdAt, + ), + ).toBe(true); + + const resumed = await manager.getJob(streamId); + expect(resumed?.status).toBe('running'); + expect(resumed?.metadata.userSubmittedPaths).toEqual(['/content/0/tool_call/args']); + expect(resumed?.metadata.userSubmittedMessageFieldPaths).toEqual( + userSubmittedMessageFieldPaths, + ); + await expect(jobStore.getJob(streamId)).resolves.toMatchObject({ + preResumeProvenance: { userSubmittedPaths: [], userSubmittedMessageFieldPaths: [] }, + }); + }); + test('returns false when the job is not paused', async () => { const streamId = 'stream-resolve-running'; await manager.createJob(streamId, 'user-1'); diff --git a/packages/api/src/stream/__tests__/steering.spec.ts b/packages/api/src/stream/__tests__/steering.spec.ts index c57539610b..e1df4e1751 100644 --- a/packages/api/src/stream/__tests__/steering.spec.ts +++ b/packages/api/src/stream/__tests__/steering.spec.ts @@ -961,6 +961,230 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () = ], }); }); + + describe('abort of a claimed approval resume', () => { + type AbortResponseMessage = { + userSubmittedPaths?: string[]; + userSubmittedMessageFieldPaths?: Array<{ path: string; field: string }>; + }; + const claimedMessageFieldPaths = [ + { path: '/content/1/tool_call/output', field: 'decision_response' as const }, + ]; + const responseMessageOf = (finalEvent: unknown) => + (finalEvent as { responseMessage?: AbortResponseMessage } | null)?.responseMessage; + + async function claimResume( + streamId: string, + overrides: { + content?: Array>; + userSubmittedPaths?: string[]; + userSubmittedMessageFieldPaths?: Array<{ + path: string; + field: 'answer' | 'decision_response' | 'decision_reason'; + }>; + } = {}, + ) { + const job = await manager.createJob(streamId, 'user-1'); + await jobStore.updateJob(streamId, { + conversationId: streamId, + createdEventEmitted: true, + responseMessageId: 'response-1', + userMessage: { messageId: 'user-1' }, + userSubmittedPaths: ['/content/0/steer'], + providerExecutionStartedId: 'exec-before-pause', + }); + const content: Array> = overrides.content ?? [ + { type: 'text', text: 'Before the pause' }, + { type: 'tool_call', tool_call: { id: 'call-1', args: '{"q":"model"}' } }, + ]; + manager.setContentParts(streamId, content as unknown as Agents.MessageContentComplex[]); + const action = { + actionId: `action-${streamId}`, + type: 'tool_approval', + payload: { action_requests: [], review_configs: [] }, + createdAt: Date.now(), + } as unknown as Agents.PendingAction; + await manager.approvals.pause(streamId, action); + const claimed = await manager.approvals.resolve( + streamId, + action.actionId, + { + providerExecutionId: `exec-${streamId}`, + providerDrained: true, + userSubmittedPaths: overrides.userSubmittedPaths ?? [ + '/content/0/steer', + '/content/2', + '/content/1/tool_call/args', + ], + userSubmittedMessageFieldPaths: + overrides.userSubmittedMessageFieldPaths ?? claimedMessageFieldPaths, + }, + job.createdAt, + ); + expect(claimed).toBe(true); + return { job, content }; + } + + test('does not label an unanswered ask placeholder as the claimed answer', async () => { + const streamId = 'steer-abort-ask-placeholder-provenance'; + await claimResume(streamId, { + content: [ + { type: 'text', text: 'Before the pause' }, + { + type: 'tool_call', + tool_call: { id: 'ask-1', name: 'ask_user_question', args: '{}', output: '' }, + }, + ], + userSubmittedPaths: ['/content/0/steer'], + userSubmittedMessageFieldPaths: [ + { path: '/content/1/tool_call/output', field: 'answer' }, + ], + }); + + const result = await manager.abortJob(streamId); + + expect(result.success).toBe(true); + const responseMessage = responseMessageOf(result.finalEvent); + expect(responseMessage?.userSubmittedPaths).toEqual(['/content/0/steer']); + expect(responseMessage).not.toHaveProperty('userSubmittedMessageFieldPaths'); + }); + + test('remaps an applied decision past a filtered part onto the published content', async () => { + const streamId = 'steer-abort-filtered-ordinal-provenance'; + const { job } = await claimResume(streamId, { + content: [ + { type: 'text', text: 'Before the pause' }, + { type: 'text', text: ' ' }, + { + type: 'tool_call', + tool_call: { id: 'call-1', args: '{"q":"user"}', output: 'User response' }, + }, + ], + userSubmittedPaths: ['/content/0/steer', '/content/1', '/content/2/tool_call/args'], + userSubmittedMessageFieldPaths: [ + { path: '/content/2/tool_call/output', field: 'decision_response' }, + ], + }); + await expect( + manager.beginProviderExecution(streamId, job.createdAt, `exec-${streamId}`), + ).resolves.toBe(true); + const persisted: unknown[] = []; + + const result = await manager.abortJob(streamId, { + beforePublish: async (pending) => { + persisted.push(pending.jobData); + }, + }); + + expect(result.success).toBe(true); + const remapped = { + userSubmittedPaths: ['/content/0/steer', '/content/1/tool_call/args'], + userSubmittedMessageFieldPaths: [ + { path: '/content/1/tool_call/output', field: 'decision_response' }, + ], + }; + expect(responseMessageOf(result.finalEvent)).toMatchObject(remapped); + expect(persisted).toEqual([expect.objectContaining(remapped)]); + }); + + test('publishes the pre-claim provenance when the provider never started', async () => { + const streamId = 'steer-abort-unapplied-provenance'; + await claimResume(streamId); + + const result = await manager.abortJob(streamId); + + expect(result.success).toBe(true); + const responseMessage = responseMessageOf(result.finalEvent); + expect(responseMessage?.userSubmittedPaths).toEqual(['/content/0/steer', '/content/2']); + expect(responseMessage).not.toHaveProperty('userSubmittedMessageFieldPaths'); + }); + + test('publishes the pre-claim provenance while the provider has started without applying the decision', async () => { + const streamId = 'steer-abort-started-unapplied-provenance'; + const { job } = await claimResume(streamId); + await expect( + manager.beginProviderExecution(streamId, job.createdAt, `exec-${streamId}`), + ).resolves.toBe(true); + + const result = await manager.abortJob(streamId); + + expect(result.success).toBe(true); + const responseMessage = responseMessageOf(result.finalEvent); + expect(responseMessage?.userSubmittedPaths).toEqual(['/content/0/steer', '/content/2']); + expect(responseMessage).not.toHaveProperty('userSubmittedMessageFieldPaths'); + }); + + test('publishes and persists the claimed provenance once the resumed tool completed', async () => { + const streamId = 'steer-abort-applied-provenance'; + const { job, content } = await claimResume(streamId); + await expect( + manager.beginProviderExecution(streamId, job.createdAt, `exec-${streamId}`), + ).resolves.toBe(true); + content[1] = { + type: 'tool_call', + tool_call: { id: 'call-1', args: '{"q":"user"}', output: 'User response' }, + }; + const persisted: unknown[] = []; + + const result = await manager.abortJob(streamId, { + beforePublish: async (pending) => { + persisted.push(pending.jobData); + }, + }); + + expect(result.success).toBe(true); + const claimedProvenance = { + userSubmittedPaths: ['/content/0/steer', '/content/2', '/content/1/tool_call/args'], + userSubmittedMessageFieldPaths: claimedMessageFieldPaths, + }; + expect(responseMessageOf(result.finalEvent)).toMatchObject(claimedProvenance); + expect(persisted).toEqual([expect.objectContaining(claimedProvenance)]); + }); + + test.each([ + ['steer-abort-stamped-provenance', 'Stamped answer'], + ['steer-abort-stamped-empty-provenance', ''], + ])( + 'labels a decision output that the abort transform stamped before the provider started (%s)', + async (streamId, output) => { + await claimResume(streamId); + + const result = await manager.abortJob(streamId, { + transformAbortContent: (parts) => + parts.map((part, index) => + index === 1 && part.type === 'tool_call' + ? { ...part, tool_call: { ...part.tool_call, output } } + : part, + ) as typeof parts, + }); + + expect(result.success).toBe(true); + expect(responseMessageOf(result.finalEvent)?.userSubmittedMessageFieldPaths).toEqual( + claimedMessageFieldPaths, + ); + }, + ); + + test('persists the same pre-claim provenance that the final event publishes', async () => { + const streamId = 'steer-abort-unapplied-persisted-provenance'; + await claimResume(streamId); + const persisted: unknown[] = []; + + const result = await manager.abortJob(streamId, { + beforePublish: async (pending) => { + persisted.push(pending.jobData); + }, + }); + + expect(result.success).toBe(true); + expect(persisted).toEqual([ + expect.objectContaining({ + userSubmittedPaths: ['/content/0/steer', '/content/2'], + userSubmittedMessageFieldPaths: [], + }), + ]); + }); + }); }); describe('synthesizeAppliedSteerEvents (snapshot→subscribe gap)', () => { diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts index d5c4cff451..0489061deb 100644 --- a/packages/api/src/stream/implementations/RedisJobStore.ts +++ b/packages/api/src/stream/implementations/RedisJobStore.ts @@ -5362,6 +5362,9 @@ export class RedisJobStore implements IJobStoreV2 { userSubmittedMessageFieldPaths: data.userSubmittedMessageFieldPaths ? JSON.parse(data.userSubmittedMessageFieldPaths) : undefined, + preResumeProvenance: data.preResumeProvenance + ? JSON.parse(data.preResumeProvenance) + : undefined, createdEventEmitted: data.createdEventEmitted === '1', sender: data.sender || undefined, syncSent: data.syncSent === '1', diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index 467e067985..3dd497b203 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -177,6 +177,13 @@ export interface SerializableJobData { userSubmittedPaths?: string[]; /** Exact request-only message fields embedded at caller-authored paths. */ userSubmittedMessageFieldPaths?: UserSubmittedMessageFieldPath[]; + /** Provenance that the latest approval claim replaced. Until that resume's provider + * segment starts, its decision is not in the job's content, so an abort publishes + * these paths instead of the claimed ones. */ + preResumeProvenance?: Pick< + SerializableJobData, + 'userSubmittedPaths' | 'userSubmittedMessageFieldPaths' + >; /** * Whether this run has activity labels enabled (per-endpoint