From 402c19ce32884f78be9c528fb732060f7a6efe8c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 24 Sep 2026 21:50:50 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20ci:=20Count=20System=20Instructi?= =?UTF-8?q?ons=20in=20Mock=20Model=20Usage=20(#16350)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e fake model reported input tokens for the chat messages alone, because the SDK hands a test override model the pruned messages without the systemRunnable pipe. Once #16301 gave every memory-enabled chat a system prompt, the calibrated context snapshot put used tokens below the instructions and the gauge dropped its Messages row. Count input over the complete prompt, as a real provider bills it. --- e2e/setup/fake-model.js | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index fbd658c5fe..51db43d721 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -698,10 +698,13 @@ function replyResponses(text) { /** * Attaches synthetic usage_metadata on a final empty chunk (the OpenAI * streaming pattern) so token-usage SSE events flow end to end in mock runs. + * Input is counted over the complete prompt a real provider bills — system + * instructions included — since the context snapshot calibrates against it. */ class UsageEmittingFakeChatModel extends FakeChatModel { - constructor({ resolveInvocation, resolveOnStream, sleep, ...options }) { + constructor({ graph, resolveInvocation, resolveOnStream, sleep, ...options }) { super({ ...options, sleep }); + this.graph = graph; this.resolveInvocation = resolveInvocation; this.resolveOnStream = resolveOnStream; this.streamSleep = sleep ?? CHUNK_DELAY_MS; @@ -818,7 +821,13 @@ class UsageEmittingFakeChatModel extends FakeChatModel { outputChars += typeof chunk.text === 'string' ? chunk.text.length : 0; yield chunk; } - const inputChars = (messages ?? []).reduce( + const { messages: promptMessages } = await getStreamAgentView({ + graph: this.graph, + messages: messages ?? [], + options, + runManager, + }); + const inputChars = promptMessages.reduce( (sum, message) => sum + getContentText(message?.content).length, 0, ); @@ -869,6 +878,7 @@ function overrideModel({ if (!thrownError) { const model = new UsageEmittingFakeChatModel({ + graph, responses, sleep: sleep ?? CHUNK_DELAY_MS, emitCustomEvent: true, @@ -2792,12 +2802,12 @@ function provisioningToolResponses({ text, toolNames }) { const command = Array.from({ length: 120 }, (_, index) => `printf 'line-${index}-☃\\n'`).join( '\n', ); - const args = - codeTool.name === 'bash_tool' - ? { command } - : codeTool.name === 'execute_code' - ? { lang: 'bash', code: command } - : codeTool.args; + let args = codeTool.args; + if (codeTool.name === 'bash_tool') { + args = { command }; + } else if (codeTool.name === 'execute_code') { + args = { lang: 'bash', code: command }; + } return { responses: ['', `E2E highlighted code complete: ${highlightLabel}`], sleep: highlightLabel === 'cancel' ? HIGHLIGHT_CANCEL_CHUNK_DELAY_MS : SLOW_CHUNK_DELAY_MS,