📎 test: Verify Stateful Sandbox Upload Continuity (#14363)

Covers the one leg the credential-free mock suite cannot fake: a file
attached through the UI's Code Environment target must land in the live
Code API, get delivered into the sandbox, and stay readable across turns.

- fake-model: E2E_EXEC_UPLOADED:/E2E_EXEC_PERSIST: markers emit real
  bash_tool calls through the production tool pipeline (the sandbox
  surface stateful agents register; execute_code stays host-wired and
  never appears in the tool registry), plus an env-gated dump of each
  run's advertised tools for future spec authoring
- spec: attach uploads a CSV to the Code API, turn 1 reads it back from
  /mnt/data and drops a proof file no upload contained, turn 2 (sent
  without an attachment) reads both — only possible when the run reuses
  the same stateful runtime session
- e2e config: agents capabilities gain stateful_code_sessions, and the
  mock profile pins CHECK_BALANCE=false so a developer's local .env can
  never fail every send with zero-balance users
- skips unless LIBRECHAT_CODE_BASEURL is provided (kept out of CI)
This commit is contained in:
Danny Avila 2026-09-09 00:35:28 -04:00 • committed by GitHub
parent dad20b115e
commit fed09dacb6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 248 additions and 2 deletions

View file

@ -111,6 +111,7 @@ endpoints:
- chain
- ocr
- run_in_background
- stateful_code_sessions
# __E2E_CODE_BRIDGE_CONFIG__
# Keep the shared mock profile non-interactive except for the dedicated
# approval probe. This exercises real HITL pause/resume without wedging the

View file

@ -127,6 +127,14 @@ const vanillaOverrides = {
CHECK_BALANCE: 'false',
};
const externalCodeBaseUrl = process.env.E2E_CODE_BASEURL?.trim();
const codeApiKeyEnv: Record<string, string> = {};
if (!externalCodeBaseUrl) {
codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = 'e2e-code-key';
} else if (process.env.E2E_CODE_API_KEY) {
codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = process.env.E2E_CODE_API_KEY;
}
const baseEnv = {
...getLocalE2EEnv(),
CONFIG_PATH: configPath,
@ -149,8 +157,9 @@ const baseEnv = {
? { E2E_CODE_BRIDGE_ADMIN_TOKEN: process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN }
: {}),
/** Point code-env + RAG provisioning at the local fakes started below. */
LIBRECHAT_CODE_BASEURL: `http://127.0.0.1:${CODE_API_PORT}/v1`,
LIBRECHAT_CODE_API_KEY: 'e2e-code-key',
LIBRECHAT_CODE_BASEURL: externalCodeBaseUrl || `http://127.0.0.1:${CODE_API_PORT}/v1`,
...(externalCodeBaseUrl ? { LIBRECHAT_CODE_BASEURL_STATEFUL: externalCodeBaseUrl } : {}),
...codeApiKeyEnv,
RAG_API_URL: `http://127.0.0.1:${RAG_API_PORT}`,
...vanillaOverrides,
};

View file

@ -104,6 +104,8 @@ const APPROVAL_TOOL_CALL_PREFIX = 'call_e2e_approval_';
const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch';
const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect';
const EXECUTE_CODE_MARKER = 'E2E_EXECUTE_CODE:';
const EXEC_UPLOADED_MARKER = 'E2E_EXEC_UPLOADED:';
const EXEC_PERSIST_MARKER = 'E2E_EXEC_PERSIST:';
const FILE_SEARCH_MARKER = 'E2E_FILE_SEARCH:';
/** Code Interpreter advertises bash_tool/read_file at runtime (execute_code is legacy);
* emit whichever the agent actually exposes so the tool batch — and provisioning — fires. */
@ -116,6 +118,11 @@ const FILE_SEARCH_TOOL_NAME = 'file_search';
const EXECUTE_CODE_FINAL_TEXT = 'E2E execute_code complete';
const FILE_SEARCH_FINAL_TEXT = 'E2E file_search complete';
const EXECUTE_CODE_TOOL_CALL_ID = 'call_e2e_execute_code';
const EXEC_UPLOADED_TOOL_CALL_ID = 'call_e2e_exec_uploaded';
const EXEC_PERSIST_TOOL_CALL_ID = 'call_e2e_exec_persist';
const EXEC_UPLOADED_FINAL_TEXT = 'E2E code exec complete';
const EXEC_PERSIST_FINAL_TEXT = 'E2E code persistence complete';
const EXEC_TURN_MARKER_FILE = 'e2e-turn1-marker.txt';
const FILE_SEARCH_TOOL_CALL_ID = 'call_e2e_file_search';
const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
@ -189,6 +196,15 @@ function getRequestedSkillName(text, marker) {
return afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0] ?? '';
}
function getRequestedSandboxFilename(text, marker) {
const markerIndex = text.indexOf(marker);
if (markerIndex === -1) {
return '';
}
const afterMarker = text.slice(markerIndex + marker.length);
return afterMarker.match(/[A-Za-z0-9][A-Za-z0-9._-]*/)?.[0] ?? '';
}
function getMarkerValue(text, marker) {
const markerIndex = text.indexOf(marker);
if (markerIndex === -1) {
@ -2304,6 +2320,32 @@ function buildHandoffResponses(graph, parsed) {
* batch to fire. Guards assert the resource tool was actually advertised.
*/
function provisioningToolResponses({ text, toolNames }) {
const uploadedFilename = getRequestedSandboxFilename(text, EXEC_UPLOADED_MARKER);
if (uploadedFilename) {
return codeExecResponses(
{
filename: uploadedFilename,
toolCallId: EXEC_UPLOADED_TOOL_CALL_ID,
finalText: EXEC_UPLOADED_FINAL_TEXT,
code: `cat "/mnt/data/${uploadedFilename}" && printf 'turn1-proof-%s\\n' "$((40 + 2))" > "/mnt/data/${EXEC_TURN_MARKER_FILE}"`,
},
toolNames,
);
}
const persistedFilename = getRequestedSandboxFilename(text, EXEC_PERSIST_MARKER);
if (persistedFilename) {
return codeExecResponses(
{
filename: persistedFilename,
toolCallId: EXEC_PERSIST_TOOL_CALL_ID,
finalText: EXEC_PERSIST_FINAL_TEXT,
code: `printf 'LINES=%s\\n' "$(wc -l < "/mnt/data/${persistedFilename}")" && cat "/mnt/data/${EXEC_TURN_MARKER_FILE}"`,
},
toolNames,
);
}
const codeLabel = getMarkerValue(text, EXECUTE_CODE_MARKER);
if (codeLabel) {
const codeTool = CODE_EXEC_TOOLS.find((tool) => toolNames.has(tool.name));
@ -2352,6 +2394,25 @@ function provisioningToolResponses({ text, toolNames }) {
return null;
}
function codeExecResponses({ filename, toolCallId, finalText, code }, toolNames) {
if (!toolNames.has(BASH_TOOL_NAME)) {
return {
responses: [`E2E code exec unavailable: ${BASH_TOOL_NAME} was not advertised.`],
};
}
return {
responses: ['', `${finalText}: ${filename}`],
toolCalls: [
{
id: toolCallId,
name: BASH_TOOL_NAME,
args: { command: code },
type: 'tool_call',
},
],
};
}
function resolveResponses({ graph, messages, text, toolNames }) {
const backgroundCompletion = backgroundCompletionResponses(text);
if (backgroundCompletion) {

View file

@ -0,0 +1,175 @@
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
import { cleanupAgent } from './agents.helpers';
import {
NEW_CHAT_PATH,
getAccessToken,
messagesView,
requestJson,
sendMessageAndWaitForCompletion,
} from './helpers';
/**
* Opt-in coverage for uploaded files reaching the code sandbox — the one leg
* the credential-free mock suite cannot fake, so it requires a live Code API
* and skips otherwise. Run with:
*
* E2E_CODE_BASEURL=http://localhost:3112/v1 \
* E2E_CODE_API_KEY=dummy \
* npx playwright test --config=e2e/playwright.config.mock.ts code-upload
*
* The fake model maps the E2E_EXEC_UPLOADED:/E2E_EXEC_PERSIST: markers to real
* `execute_code` bash calls, so everything between the browser and the sandbox
* is the production path: attaching the file uploads it to the Code API,
* turn 1's exec reads it back from /mnt/data and drops a marker file no upload
* ever contained, and turn 2 (sent WITHOUT an attachment) reads both — which
* only works when the run reuses the same stateful runtime session.
*/
const CODE_BASEURL = process.env.E2E_CODE_BASEURL?.trim() ?? '';
const FILE_NAME = 'e2e-sandbox-data.csv';
const FILE_CONTENT = 'name,value\nalpha,1\nbeta,2\n';
const EXEC_FINAL_TEXT = 'E2E code exec complete';
const PERSIST_FINAL_TEXT = 'E2E code persistence complete';
/* Produced only by sandbox command OUTPUT (the fake model's commands keep
* these strings out of the tool args via printf formats), so matching the
* conversation transcript cannot be satisfied by the commands themselves. */
const TURN1_OUTPUT_NEEDLE = 'alpha,1';
const TURN2_LINES_NEEDLE = 'LINES=3';
const TURN2_MARKER_NEEDLE = 'turn1-proof-42';
const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
const modelTrigger = (page: Page) => page.getByRole('button', { name: 'Select a model' }).first();
async function startFresh(page: Page) {
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
await page.evaluate(() => localStorage.clear());
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
}
type AgentResponse = { id: string };
async function createCodeAgent(page: Page, name: string): Promise<AgentResponse> {
const token = await getAccessToken(page);
return requestJson<AgentResponse>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
name,
provider: 'Mock Provider A',
model: 'mock-model-a',
model_parameters: {},
tools: ['execute_code'],
stateful_code_sessions: true,
stateful_code_environment: 'conversation',
},
});
}
async function selectAgent(page: Page, agentName: string) {
await modelTrigger(page).click();
await page.getByRole('option', { name: 'My Agents' }).click();
await page.getByRole('option', { name: agentName }).click();
await expect(modelTrigger(page)).toContainText(agentName);
}
/** Attaches a CSV through the real attach menu → Code Environment target,
* which uploads it to the live Code API before the message is ever sent. */
async function attachCodeFile(page: Page) {
const [chooser] = await Promise.all([
page.waitForEvent('filechooser'),
(async () => {
await page.getByRole('button', { name: 'Attach File Options' }).click();
await page.getByRole('menuitem', { name: 'Upload to Code Environment' }).click();
})(),
]);
const [upload] = await Promise.all([
page.waitForResponse(
(response) =>
new URL(response.url()).pathname === '/api/files' && response.request().method() === 'POST',
{ timeout: 60_000 },
),
chooser.setFiles({
name: FILE_NAME,
mimeType: 'text/csv',
buffer: Buffer.from(FILE_CONTENT),
}),
]);
expect(upload.ok(), `code file upload returned ${upload.status()}`).toBeTruthy();
await expect(page.getByText(FILE_NAME).first()).toBeVisible();
}
async function conversationIncludes(
page: Page,
conversationId: string,
needle: string,
): Promise<boolean> {
const token = await getAccessToken(page);
const messages = await requestJson<unknown[]>(page, {
path: `/api/messages/${encodeURIComponent(conversationId)}`,
token,
});
return JSON.stringify(messages).includes(needle);
}
test.describe('stateful code sandbox uploads', () => {
test.skip(
!CODE_BASEURL,
'Requires a live Code API: set E2E_CODE_BASEURL (and optionally E2E_CODE_API_KEY)',
);
test('uploaded file reaches /mnt/data and the session persists across turns', async ({
page,
}) => {
/* Two real sandbox execs, each possibly paying a cold VM boot. */
test.setTimeout(600_000);
await startFresh(page);
const agentName = uniqueName('E2E Code Sandbox');
let agentId: string | undefined;
try {
const agent = await createCodeAgent(page, agentName);
agentId = agent.id;
await selectAgent(page, agentName);
await attachCodeFile(page);
await sendMessageAndWaitForCompletion(page, `E2E_EXEC_UPLOADED:${FILE_NAME}`, {
timeout: 180_000,
});
await expect(messagesView(page).getByText(`${EXEC_FINAL_TEXT}: ${FILE_NAME}`)).toBeVisible();
await page.waitForURL(/\/c\/(?!new$)[^/?]+/, { timeout: 10_000 });
const conversationId = new URL(page.url()).pathname.split('/').pop() as string;
/* The exec's stdout — the uploaded bytes read back from /mnt/data. */
await expect
.poll(() => conversationIncludes(page, conversationId, TURN1_OUTPUT_NEEDLE), {
timeout: 30_000,
})
.toBe(true);
/* Turn 2 carries NO attachment: the line count re-reads the uploaded file
* and the marker file exists only inside the prior turn's session. */
await sendMessageAndWaitForCompletion(page, `E2E_EXEC_PERSIST:${FILE_NAME}`, {
timeout: 180_000,
});
await expect(
messagesView(page).getByText(`${PERSIST_FINAL_TEXT}: ${FILE_NAME}`),
).toBeVisible();
await expect
.poll(() => conversationIncludes(page, conversationId, TURN2_LINES_NEEDLE), {
timeout: 30_000,
})
.toBe(true);
await expect
.poll(() => conversationIncludes(page, conversationId, TURN2_MARKER_NEEDLE), {
timeout: 30_000,
})
.toBe(true);
} finally {
await cleanupAgent(page, agentId);
}
});
});