mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
📎 test: Verify Stateful Sandbox Upload Continuity (#14363)
Covers the one leg the credential-free mock suite cannot fake: a file attached through the UI's Code Environment target must land in the live Code API, get delivered into the sandbox, and stay readable across turns. - fake-model: E2E_EXEC_UPLOADED:/E2E_EXEC_PERSIST: markers emit real bash_tool calls through the production tool pipeline (the sandbox surface stateful agents register; execute_code stays host-wired and never appears in the tool registry), plus an env-gated dump of each run's advertised tools for future spec authoring - spec: attach uploads a CSV to the Code API, turn 1 reads it back from /mnt/data and drops a proof file no upload contained, turn 2 (sent without an attachment) reads both — only possible when the run reuses the same stateful runtime session - e2e config: agents capabilities gain stateful_code_sessions, and the mock profile pins CHECK_BALANCE=false so a developer's local .env can never fail every send with zero-balance users - skips unless LIBRECHAT_CODE_BASEURL is provided (kept out of CI)
This commit is contained in:
parent
dad20b115e
commit
fed09dacb6
4 changed files with 248 additions and 2 deletions
|
|
@ -111,6 +111,7 @@ endpoints:
|
|||
- chain
|
||||
- ocr
|
||||
- run_in_background
|
||||
- stateful_code_sessions
|
||||
# __E2E_CODE_BRIDGE_CONFIG__
|
||||
# Keep the shared mock profile non-interactive except for the dedicated
|
||||
# approval probe. This exercises real HITL pause/resume without wedging the
|
||||
|
|
|
|||
|
|
@ -127,6 +127,14 @@ const vanillaOverrides = {
|
|||
CHECK_BALANCE: 'false',
|
||||
};
|
||||
|
||||
const externalCodeBaseUrl = process.env.E2E_CODE_BASEURL?.trim();
|
||||
const codeApiKeyEnv: Record<string, string> = {};
|
||||
if (!externalCodeBaseUrl) {
|
||||
codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = 'e2e-code-key';
|
||||
} else if (process.env.E2E_CODE_API_KEY) {
|
||||
codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = process.env.E2E_CODE_API_KEY;
|
||||
}
|
||||
|
||||
const baseEnv = {
|
||||
...getLocalE2EEnv(),
|
||||
CONFIG_PATH: configPath,
|
||||
|
|
@ -149,8 +157,9 @@ const baseEnv = {
|
|||
? { E2E_CODE_BRIDGE_ADMIN_TOKEN: process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN }
|
||||
: {}),
|
||||
/** Point code-env + RAG provisioning at the local fakes started below. */
|
||||
LIBRECHAT_CODE_BASEURL: `http://127.0.0.1:${CODE_API_PORT}/v1`,
|
||||
LIBRECHAT_CODE_API_KEY: 'e2e-code-key',
|
||||
LIBRECHAT_CODE_BASEURL: externalCodeBaseUrl || `http://127.0.0.1:${CODE_API_PORT}/v1`,
|
||||
...(externalCodeBaseUrl ? { LIBRECHAT_CODE_BASEURL_STATEFUL: externalCodeBaseUrl } : {}),
|
||||
...codeApiKeyEnv,
|
||||
RAG_API_URL: `http://127.0.0.1:${RAG_API_PORT}`,
|
||||
...vanillaOverrides,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -104,6 +104,8 @@ const APPROVAL_TOOL_CALL_PREFIX = 'call_e2e_approval_';
|
|||
const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch';
|
||||
const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect';
|
||||
const EXECUTE_CODE_MARKER = 'E2E_EXECUTE_CODE:';
|
||||
const EXEC_UPLOADED_MARKER = 'E2E_EXEC_UPLOADED:';
|
||||
const EXEC_PERSIST_MARKER = 'E2E_EXEC_PERSIST:';
|
||||
const FILE_SEARCH_MARKER = 'E2E_FILE_SEARCH:';
|
||||
/** Code Interpreter advertises bash_tool/read_file at runtime (execute_code is legacy);
|
||||
* emit whichever the agent actually exposes so the tool batch — and provisioning — fires. */
|
||||
|
|
@ -116,6 +118,11 @@ const FILE_SEARCH_TOOL_NAME = 'file_search';
|
|||
const EXECUTE_CODE_FINAL_TEXT = 'E2E execute_code complete';
|
||||
const FILE_SEARCH_FINAL_TEXT = 'E2E file_search complete';
|
||||
const EXECUTE_CODE_TOOL_CALL_ID = 'call_e2e_execute_code';
|
||||
const EXEC_UPLOADED_TOOL_CALL_ID = 'call_e2e_exec_uploaded';
|
||||
const EXEC_PERSIST_TOOL_CALL_ID = 'call_e2e_exec_persist';
|
||||
const EXEC_UPLOADED_FINAL_TEXT = 'E2E code exec complete';
|
||||
const EXEC_PERSIST_FINAL_TEXT = 'E2E code persistence complete';
|
||||
const EXEC_TURN_MARKER_FILE = 'e2e-turn1-marker.txt';
|
||||
const FILE_SEARCH_TOOL_CALL_ID = 'call_e2e_file_search';
|
||||
const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
|
||||
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
|
||||
|
|
@ -189,6 +196,15 @@ function getRequestedSkillName(text, marker) {
|
|||
return afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0] ?? '';
|
||||
}
|
||||
|
||||
function getRequestedSandboxFilename(text, marker) {
|
||||
const markerIndex = text.indexOf(marker);
|
||||
if (markerIndex === -1) {
|
||||
return '';
|
||||
}
|
||||
const afterMarker = text.slice(markerIndex + marker.length);
|
||||
return afterMarker.match(/[A-Za-z0-9][A-Za-z0-9._-]*/)?.[0] ?? '';
|
||||
}
|
||||
|
||||
function getMarkerValue(text, marker) {
|
||||
const markerIndex = text.indexOf(marker);
|
||||
if (markerIndex === -1) {
|
||||
|
|
@ -2304,6 +2320,32 @@ function buildHandoffResponses(graph, parsed) {
|
|||
* batch to fire. Guards assert the resource tool was actually advertised.
|
||||
*/
|
||||
function provisioningToolResponses({ text, toolNames }) {
|
||||
const uploadedFilename = getRequestedSandboxFilename(text, EXEC_UPLOADED_MARKER);
|
||||
if (uploadedFilename) {
|
||||
return codeExecResponses(
|
||||
{
|
||||
filename: uploadedFilename,
|
||||
toolCallId: EXEC_UPLOADED_TOOL_CALL_ID,
|
||||
finalText: EXEC_UPLOADED_FINAL_TEXT,
|
||||
code: `cat "/mnt/data/${uploadedFilename}" && printf 'turn1-proof-%s\\n' "$((40 + 2))" > "/mnt/data/${EXEC_TURN_MARKER_FILE}"`,
|
||||
},
|
||||
toolNames,
|
||||
);
|
||||
}
|
||||
|
||||
const persistedFilename = getRequestedSandboxFilename(text, EXEC_PERSIST_MARKER);
|
||||
if (persistedFilename) {
|
||||
return codeExecResponses(
|
||||
{
|
||||
filename: persistedFilename,
|
||||
toolCallId: EXEC_PERSIST_TOOL_CALL_ID,
|
||||
finalText: EXEC_PERSIST_FINAL_TEXT,
|
||||
code: `printf 'LINES=%s\\n' "$(wc -l < "/mnt/data/${persistedFilename}")" && cat "/mnt/data/${EXEC_TURN_MARKER_FILE}"`,
|
||||
},
|
||||
toolNames,
|
||||
);
|
||||
}
|
||||
|
||||
const codeLabel = getMarkerValue(text, EXECUTE_CODE_MARKER);
|
||||
if (codeLabel) {
|
||||
const codeTool = CODE_EXEC_TOOLS.find((tool) => toolNames.has(tool.name));
|
||||
|
|
@ -2352,6 +2394,25 @@ function provisioningToolResponses({ text, toolNames }) {
|
|||
return null;
|
||||
}
|
||||
|
||||
function codeExecResponses({ filename, toolCallId, finalText, code }, toolNames) {
|
||||
if (!toolNames.has(BASH_TOOL_NAME)) {
|
||||
return {
|
||||
responses: [`E2E code exec unavailable: ${BASH_TOOL_NAME} was not advertised.`],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: ['', `${finalText}: ${filename}`],
|
||||
toolCalls: [
|
||||
{
|
||||
id: toolCallId,
|
||||
name: BASH_TOOL_NAME,
|
||||
args: { command: code },
|
||||
type: 'tool_call',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function resolveResponses({ graph, messages, text, toolNames }) {
|
||||
const backgroundCompletion = backgroundCompletionResponses(text);
|
||||
if (backgroundCompletion) {
|
||||
|
|
|
|||
175
e2e/specs/mock/code-upload.spec.ts
Normal file
175
e2e/specs/mock/code-upload.spec.ts
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import { cleanupAgent } from './agents.helpers';
|
||||
import {
|
||||
NEW_CHAT_PATH,
|
||||
getAccessToken,
|
||||
messagesView,
|
||||
requestJson,
|
||||
sendMessageAndWaitForCompletion,
|
||||
} from './helpers';
|
||||
|
||||
/**
|
||||
* Opt-in coverage for uploaded files reaching the code sandbox — the one leg
|
||||
* the credential-free mock suite cannot fake, so it requires a live Code API
|
||||
* and skips otherwise. Run with:
|
||||
*
|
||||
* E2E_CODE_BASEURL=http://localhost:3112/v1 \
|
||||
* E2E_CODE_API_KEY=dummy \
|
||||
* npx playwright test --config=e2e/playwright.config.mock.ts code-upload
|
||||
*
|
||||
* The fake model maps the E2E_EXEC_UPLOADED:/E2E_EXEC_PERSIST: markers to real
|
||||
* `execute_code` bash calls, so everything between the browser and the sandbox
|
||||
* is the production path: attaching the file uploads it to the Code API,
|
||||
* turn 1's exec reads it back from /mnt/data and drops a marker file no upload
|
||||
* ever contained, and turn 2 (sent WITHOUT an attachment) reads both — which
|
||||
* only works when the run reuses the same stateful runtime session.
|
||||
*/
|
||||
|
||||
const CODE_BASEURL = process.env.E2E_CODE_BASEURL?.trim() ?? '';
|
||||
const FILE_NAME = 'e2e-sandbox-data.csv';
|
||||
const FILE_CONTENT = 'name,value\nalpha,1\nbeta,2\n';
|
||||
const EXEC_FINAL_TEXT = 'E2E code exec complete';
|
||||
const PERSIST_FINAL_TEXT = 'E2E code persistence complete';
|
||||
/* Produced only by sandbox command OUTPUT (the fake model's commands keep
|
||||
* these strings out of the tool args via printf formats), so matching the
|
||||
* conversation transcript cannot be satisfied by the commands themselves. */
|
||||
const TURN1_OUTPUT_NEEDLE = 'alpha,1';
|
||||
const TURN2_LINES_NEEDLE = 'LINES=3';
|
||||
const TURN2_MARKER_NEEDLE = 'turn1-proof-42';
|
||||
|
||||
const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
|
||||
const modelTrigger = (page: Page) => page.getByRole('button', { name: 'Select a model' }).first();
|
||||
|
||||
async function startFresh(page: Page) {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
}
|
||||
|
||||
type AgentResponse = { id: string };
|
||||
|
||||
async function createCodeAgent(page: Page, name: string): Promise<AgentResponse> {
|
||||
const token = await getAccessToken(page);
|
||||
return requestJson<AgentResponse>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name,
|
||||
provider: 'Mock Provider A',
|
||||
model: 'mock-model-a',
|
||||
model_parameters: {},
|
||||
tools: ['execute_code'],
|
||||
stateful_code_sessions: true,
|
||||
stateful_code_environment: 'conversation',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function selectAgent(page: Page, agentName: string) {
|
||||
await modelTrigger(page).click();
|
||||
await page.getByRole('option', { name: 'My Agents' }).click();
|
||||
await page.getByRole('option', { name: agentName }).click();
|
||||
await expect(modelTrigger(page)).toContainText(agentName);
|
||||
}
|
||||
|
||||
/** Attaches a CSV through the real attach menu → Code Environment target,
|
||||
* which uploads it to the live Code API before the message is ever sent. */
|
||||
async function attachCodeFile(page: Page) {
|
||||
const [chooser] = await Promise.all([
|
||||
page.waitForEvent('filechooser'),
|
||||
(async () => {
|
||||
await page.getByRole('button', { name: 'Attach File Options' }).click();
|
||||
await page.getByRole('menuitem', { name: 'Upload to Code Environment' }).click();
|
||||
})(),
|
||||
]);
|
||||
const [upload] = await Promise.all([
|
||||
page.waitForResponse(
|
||||
(response) =>
|
||||
new URL(response.url()).pathname === '/api/files' && response.request().method() === 'POST',
|
||||
{ timeout: 60_000 },
|
||||
),
|
||||
chooser.setFiles({
|
||||
name: FILE_NAME,
|
||||
mimeType: 'text/csv',
|
||||
buffer: Buffer.from(FILE_CONTENT),
|
||||
}),
|
||||
]);
|
||||
expect(upload.ok(), `code file upload returned ${upload.status()}`).toBeTruthy();
|
||||
await expect(page.getByText(FILE_NAME).first()).toBeVisible();
|
||||
}
|
||||
|
||||
async function conversationIncludes(
|
||||
page: Page,
|
||||
conversationId: string,
|
||||
needle: string,
|
||||
): Promise<boolean> {
|
||||
const token = await getAccessToken(page);
|
||||
const messages = await requestJson<unknown[]>(page, {
|
||||
path: `/api/messages/${encodeURIComponent(conversationId)}`,
|
||||
token,
|
||||
});
|
||||
return JSON.stringify(messages).includes(needle);
|
||||
}
|
||||
|
||||
test.describe('stateful code sandbox uploads', () => {
|
||||
test.skip(
|
||||
!CODE_BASEURL,
|
||||
'Requires a live Code API: set E2E_CODE_BASEURL (and optionally E2E_CODE_API_KEY)',
|
||||
);
|
||||
|
||||
test('uploaded file reaches /mnt/data and the session persists across turns', async ({
|
||||
page,
|
||||
}) => {
|
||||
/* Two real sandbox execs, each possibly paying a cold VM boot. */
|
||||
test.setTimeout(600_000);
|
||||
|
||||
await startFresh(page);
|
||||
const agentName = uniqueName('E2E Code Sandbox');
|
||||
let agentId: string | undefined;
|
||||
try {
|
||||
const agent = await createCodeAgent(page, agentName);
|
||||
agentId = agent.id;
|
||||
await selectAgent(page, agentName);
|
||||
|
||||
await attachCodeFile(page);
|
||||
await sendMessageAndWaitForCompletion(page, `E2E_EXEC_UPLOADED:${FILE_NAME}`, {
|
||||
timeout: 180_000,
|
||||
});
|
||||
await expect(messagesView(page).getByText(`${EXEC_FINAL_TEXT}: ${FILE_NAME}`)).toBeVisible();
|
||||
|
||||
await page.waitForURL(/\/c\/(?!new$)[^/?]+/, { timeout: 10_000 });
|
||||
const conversationId = new URL(page.url()).pathname.split('/').pop() as string;
|
||||
|
||||
/* The exec's stdout — the uploaded bytes read back from /mnt/data. */
|
||||
await expect
|
||||
.poll(() => conversationIncludes(page, conversationId, TURN1_OUTPUT_NEEDLE), {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toBe(true);
|
||||
|
||||
/* Turn 2 carries NO attachment: the line count re-reads the uploaded file
|
||||
* and the marker file exists only inside the prior turn's session. */
|
||||
await sendMessageAndWaitForCompletion(page, `E2E_EXEC_PERSIST:${FILE_NAME}`, {
|
||||
timeout: 180_000,
|
||||
});
|
||||
await expect(
|
||||
messagesView(page).getByText(`${PERSIST_FINAL_TEXT}: ${FILE_NAME}`),
|
||||
).toBeVisible();
|
||||
|
||||
await expect
|
||||
.poll(() => conversationIncludes(page, conversationId, TURN2_LINES_NEEDLE), {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toBe(true);
|
||||
await expect
|
||||
.poll(() => conversationIncludes(page, conversationId, TURN2_MARKER_NEEDLE), {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toBe(true);
|
||||
} finally {
|
||||
await cleanupAgent(page, agentId);
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue