From fed09dacb6ebbc79eca2b85bfa2c04170495af6a Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 9 Sep 2026 00:35:28 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=8E=20test:=20Verify=20Stateful=20Sand?= =?UTF-8?q?box=20Upload=20Continuity=20(#14363)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the one leg the credential-free mock suite cannot fake: a file attached through the UI's Code Environment target must land in the live Code API, get delivered into the sandbox, and stay readable across turns. - fake-model: E2E_EXEC_UPLOADED:/E2E_EXEC_PERSIST: markers emit real bash_tool calls through the production tool pipeline (the sandbox surface stateful agents register; execute_code stays host-wired and never appears in the tool registry), plus an env-gated dump of each run's advertised tools for future spec authoring - spec: attach uploads a CSV to the Code API, turn 1 reads it back from /mnt/data and drops a proof file no upload contained, turn 2 (sent without an attachment) reads both — only possible when the run reuses the same stateful runtime session - e2e config: agents capabilities gain stateful_code_sessions, and the mock profile pins CHECK_BALANCE=false so a developer's local .env can never fail every send with zero-balance users - skips unless LIBRECHAT_CODE_BASEURL is provided (kept out of CI) --- e2e/config/librechat.e2e.yaml | 1 + e2e/playwright.config.mock.ts | 13 ++- e2e/setup/fake-model.js | 61 ++++++++++ e2e/specs/mock/code-upload.spec.ts | 175 +++++++++++++++++++++++++++++ 4 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 e2e/specs/mock/code-upload.spec.ts diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index 8d3dbda7f2..dea966836d 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -111,6 +111,7 @@ endpoints: - chain - ocr - run_in_background + - stateful_code_sessions # __E2E_CODE_BRIDGE_CONFIG__ # Keep the shared mock profile non-interactive except for the dedicated # approval probe. This exercises real HITL pause/resume without wedging the diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index a3bbe50b2a..c51ea71d81 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -127,6 +127,14 @@ const vanillaOverrides = { CHECK_BALANCE: 'false', }; +const externalCodeBaseUrl = process.env.E2E_CODE_BASEURL?.trim(); +const codeApiKeyEnv: Record = {}; +if (!externalCodeBaseUrl) { + codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = 'e2e-code-key'; +} else if (process.env.E2E_CODE_API_KEY) { + codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = process.env.E2E_CODE_API_KEY; +} + const baseEnv = { ...getLocalE2EEnv(), CONFIG_PATH: configPath, @@ -149,8 +157,9 @@ const baseEnv = { ? { E2E_CODE_BRIDGE_ADMIN_TOKEN: process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN } : {}), /** Point code-env + RAG provisioning at the local fakes started below. */ - LIBRECHAT_CODE_BASEURL: `http://127.0.0.1:${CODE_API_PORT}/v1`, - LIBRECHAT_CODE_API_KEY: 'e2e-code-key', + LIBRECHAT_CODE_BASEURL: externalCodeBaseUrl || `http://127.0.0.1:${CODE_API_PORT}/v1`, + ...(externalCodeBaseUrl ? { LIBRECHAT_CODE_BASEURL_STATEFUL: externalCodeBaseUrl } : {}), + ...codeApiKeyEnv, RAG_API_URL: `http://127.0.0.1:${RAG_API_PORT}`, ...vanillaOverrides, }; diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index c80ef33942..859836d2df 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -104,6 +104,8 @@ const APPROVAL_TOOL_CALL_PREFIX = 'call_e2e_approval_'; const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch'; const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect'; const EXECUTE_CODE_MARKER = 'E2E_EXECUTE_CODE:'; +const EXEC_UPLOADED_MARKER = 'E2E_EXEC_UPLOADED:'; +const EXEC_PERSIST_MARKER = 'E2E_EXEC_PERSIST:'; const FILE_SEARCH_MARKER = 'E2E_FILE_SEARCH:'; /** Code Interpreter advertises bash_tool/read_file at runtime (execute_code is legacy); * emit whichever the agent actually exposes so the tool batch — and provisioning — fires. */ @@ -116,6 +118,11 @@ const FILE_SEARCH_TOOL_NAME = 'file_search'; const EXECUTE_CODE_FINAL_TEXT = 'E2E execute_code complete'; const FILE_SEARCH_FINAL_TEXT = 'E2E file_search complete'; const EXECUTE_CODE_TOOL_CALL_ID = 'call_e2e_execute_code'; +const EXEC_UPLOADED_TOOL_CALL_ID = 'call_e2e_exec_uploaded'; +const EXEC_PERSIST_TOOL_CALL_ID = 'call_e2e_exec_persist'; +const EXEC_UPLOADED_FINAL_TEXT = 'E2E code exec complete'; +const EXEC_PERSIST_FINAL_TEXT = 'E2E code persistence complete'; +const EXEC_TURN_MARKER_FILE = 'e2e-turn1-marker.txt'; const FILE_SEARCH_TOOL_CALL_ID = 'call_e2e_file_search'; const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed'; const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill'; @@ -189,6 +196,15 @@ function getRequestedSkillName(text, marker) { return afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0] ?? ''; } +function getRequestedSandboxFilename(text, marker) { + const markerIndex = text.indexOf(marker); + if (markerIndex === -1) { + return ''; + } + const afterMarker = text.slice(markerIndex + marker.length); + return afterMarker.match(/[A-Za-z0-9][A-Za-z0-9._-]*/)?.[0] ?? ''; +} + function getMarkerValue(text, marker) { const markerIndex = text.indexOf(marker); if (markerIndex === -1) { @@ -2304,6 +2320,32 @@ function buildHandoffResponses(graph, parsed) { * batch to fire. Guards assert the resource tool was actually advertised. */ function provisioningToolResponses({ text, toolNames }) { + const uploadedFilename = getRequestedSandboxFilename(text, EXEC_UPLOADED_MARKER); + if (uploadedFilename) { + return codeExecResponses( + { + filename: uploadedFilename, + toolCallId: EXEC_UPLOADED_TOOL_CALL_ID, + finalText: EXEC_UPLOADED_FINAL_TEXT, + code: `cat "/mnt/data/${uploadedFilename}" && printf 'turn1-proof-%s\\n' "$((40 + 2))" > "/mnt/data/${EXEC_TURN_MARKER_FILE}"`, + }, + toolNames, + ); + } + + const persistedFilename = getRequestedSandboxFilename(text, EXEC_PERSIST_MARKER); + if (persistedFilename) { + return codeExecResponses( + { + filename: persistedFilename, + toolCallId: EXEC_PERSIST_TOOL_CALL_ID, + finalText: EXEC_PERSIST_FINAL_TEXT, + code: `printf 'LINES=%s\\n' "$(wc -l < "/mnt/data/${persistedFilename}")" && cat "/mnt/data/${EXEC_TURN_MARKER_FILE}"`, + }, + toolNames, + ); + } + const codeLabel = getMarkerValue(text, EXECUTE_CODE_MARKER); if (codeLabel) { const codeTool = CODE_EXEC_TOOLS.find((tool) => toolNames.has(tool.name)); @@ -2352,6 +2394,25 @@ function provisioningToolResponses({ text, toolNames }) { return null; } +function codeExecResponses({ filename, toolCallId, finalText, code }, toolNames) { + if (!toolNames.has(BASH_TOOL_NAME)) { + return { + responses: [`E2E code exec unavailable: ${BASH_TOOL_NAME} was not advertised.`], + }; + } + return { + responses: ['', `${finalText}: ${filename}`], + toolCalls: [ + { + id: toolCallId, + name: BASH_TOOL_NAME, + args: { command: code }, + type: 'tool_call', + }, + ], + }; +} + function resolveResponses({ graph, messages, text, toolNames }) { const backgroundCompletion = backgroundCompletionResponses(text); if (backgroundCompletion) { diff --git a/e2e/specs/mock/code-upload.spec.ts b/e2e/specs/mock/code-upload.spec.ts new file mode 100644 index 0000000000..8628c4aa05 --- /dev/null +++ b/e2e/specs/mock/code-upload.spec.ts @@ -0,0 +1,175 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import { cleanupAgent } from './agents.helpers'; +import { + NEW_CHAT_PATH, + getAccessToken, + messagesView, + requestJson, + sendMessageAndWaitForCompletion, +} from './helpers'; + +/** + * Opt-in coverage for uploaded files reaching the code sandbox — the one leg + * the credential-free mock suite cannot fake, so it requires a live Code API + * and skips otherwise. Run with: + * + * E2E_CODE_BASEURL=http://localhost:3112/v1 \ + * E2E_CODE_API_KEY=dummy \ + * npx playwright test --config=e2e/playwright.config.mock.ts code-upload + * + * The fake model maps the E2E_EXEC_UPLOADED:/E2E_EXEC_PERSIST: markers to real + * `execute_code` bash calls, so everything between the browser and the sandbox + * is the production path: attaching the file uploads it to the Code API, + * turn 1's exec reads it back from /mnt/data and drops a marker file no upload + * ever contained, and turn 2 (sent WITHOUT an attachment) reads both — which + * only works when the run reuses the same stateful runtime session. + */ + +const CODE_BASEURL = process.env.E2E_CODE_BASEURL?.trim() ?? ''; +const FILE_NAME = 'e2e-sandbox-data.csv'; +const FILE_CONTENT = 'name,value\nalpha,1\nbeta,2\n'; +const EXEC_FINAL_TEXT = 'E2E code exec complete'; +const PERSIST_FINAL_TEXT = 'E2E code persistence complete'; +/* Produced only by sandbox command OUTPUT (the fake model's commands keep + * these strings out of the tool args via printf formats), so matching the + * conversation transcript cannot be satisfied by the commands themselves. */ +const TURN1_OUTPUT_NEEDLE = 'alpha,1'; +const TURN2_LINES_NEEDLE = 'LINES=3'; +const TURN2_MARKER_NEEDLE = 'turn1-proof-42'; + +const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`; +const modelTrigger = (page: Page) => page.getByRole('button', { name: 'Select a model' }).first(); + +async function startFresh(page: Page) { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + await page.evaluate(() => localStorage.clear()); + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); +} + +type AgentResponse = { id: string }; + +async function createCodeAgent(page: Page, name: string): Promise { + const token = await getAccessToken(page); + return requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name, + provider: 'Mock Provider A', + model: 'mock-model-a', + model_parameters: {}, + tools: ['execute_code'], + stateful_code_sessions: true, + stateful_code_environment: 'conversation', + }, + }); +} + +async function selectAgent(page: Page, agentName: string) { + await modelTrigger(page).click(); + await page.getByRole('option', { name: 'My Agents' }).click(); + await page.getByRole('option', { name: agentName }).click(); + await expect(modelTrigger(page)).toContainText(agentName); +} + +/** Attaches a CSV through the real attach menu → Code Environment target, + * which uploads it to the live Code API before the message is ever sent. */ +async function attachCodeFile(page: Page) { + const [chooser] = await Promise.all([ + page.waitForEvent('filechooser'), + (async () => { + await page.getByRole('button', { name: 'Attach File Options' }).click(); + await page.getByRole('menuitem', { name: 'Upload to Code Environment' }).click(); + })(), + ]); + const [upload] = await Promise.all([ + page.waitForResponse( + (response) => + new URL(response.url()).pathname === '/api/files' && response.request().method() === 'POST', + { timeout: 60_000 }, + ), + chooser.setFiles({ + name: FILE_NAME, + mimeType: 'text/csv', + buffer: Buffer.from(FILE_CONTENT), + }), + ]); + expect(upload.ok(), `code file upload returned ${upload.status()}`).toBeTruthy(); + await expect(page.getByText(FILE_NAME).first()).toBeVisible(); +} + +async function conversationIncludes( + page: Page, + conversationId: string, + needle: string, +): Promise { + const token = await getAccessToken(page); + const messages = await requestJson(page, { + path: `/api/messages/${encodeURIComponent(conversationId)}`, + token, + }); + return JSON.stringify(messages).includes(needle); +} + +test.describe('stateful code sandbox uploads', () => { + test.skip( + !CODE_BASEURL, + 'Requires a live Code API: set E2E_CODE_BASEURL (and optionally E2E_CODE_API_KEY)', + ); + + test('uploaded file reaches /mnt/data and the session persists across turns', async ({ + page, + }) => { + /* Two real sandbox execs, each possibly paying a cold VM boot. */ + test.setTimeout(600_000); + + await startFresh(page); + const agentName = uniqueName('E2E Code Sandbox'); + let agentId: string | undefined; + try { + const agent = await createCodeAgent(page, agentName); + agentId = agent.id; + await selectAgent(page, agentName); + + await attachCodeFile(page); + await sendMessageAndWaitForCompletion(page, `E2E_EXEC_UPLOADED:${FILE_NAME}`, { + timeout: 180_000, + }); + await expect(messagesView(page).getByText(`${EXEC_FINAL_TEXT}: ${FILE_NAME}`)).toBeVisible(); + + await page.waitForURL(/\/c\/(?!new$)[^/?]+/, { timeout: 10_000 }); + const conversationId = new URL(page.url()).pathname.split('/').pop() as string; + + /* The exec's stdout — the uploaded bytes read back from /mnt/data. */ + await expect + .poll(() => conversationIncludes(page, conversationId, TURN1_OUTPUT_NEEDLE), { + timeout: 30_000, + }) + .toBe(true); + + /* Turn 2 carries NO attachment: the line count re-reads the uploaded file + * and the marker file exists only inside the prior turn's session. */ + await sendMessageAndWaitForCompletion(page, `E2E_EXEC_PERSIST:${FILE_NAME}`, { + timeout: 180_000, + }); + await expect( + messagesView(page).getByText(`${PERSIST_FINAL_TEXT}: ${FILE_NAME}`), + ).toBeVisible(); + + await expect + .poll(() => conversationIncludes(page, conversationId, TURN2_LINES_NEEDLE), { + timeout: 30_000, + }) + .toBe(true); + await expect + .poll(() => conversationIncludes(page, conversationId, TURN2_MARKER_NEEDLE), { + timeout: 30_000, + }) + .toBe(true); + } finally { + await cleanupAgent(page, agentId); + } + }); +});