diff --git a/.github/workflows/playwright-bombadil.yml b/.github/workflows/playwright-bombadil.yml new file mode 100644 index 0000000000..040a8c4848 --- /dev/null +++ b/.github/workflows/playwright-bombadil.yml @@ -0,0 +1,180 @@ +name: Bombadil Property Exploration + +on: + pull_request: + paths: + - '**' + - '!**.md' + - '!.github/workflows/**' + - '.github/workflows/playwright-bombadil.yml' + workflow_dispatch: + inputs: + reason: + description: 'Reason for manual trigger' + required: false + default: 'Manual Bombadil run' + +permissions: + contents: read + +concurrency: + group: playwright-bombadil-${{ github.ref }} + cancel-in-progress: true + +env: + NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}' + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1' + +jobs: + bombadil: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + github.event.pull_request != null && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)) + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + BOMBADIL_TIME_LIMIT: '300s' + E2E_CHROMIUM_CHANNEL: chrome + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 24.16.0 + uses: actions/setup-node@v4 + with: + node-version: '24.16.0' + + - name: Restore node_modules cache + id: cache-node-modules + uses: actions/cache@v4 + with: + path: | + node_modules + client/node_modules + packages/client/node_modules + packages/data-provider/node_modules + packages/data-schemas/node_modules + packages/api/node_modules + api/node_modules + key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci + + - name: Restore data-provider build cache + id: cache-data-provider + uses: actions/cache@v4 + with: + path: packages/data-provider/dist + key: build-data-provider-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }} + + - name: Build data-provider + if: steps.cache-data-provider.outputs.cache-hit != 'true' + run: npm run build:data-provider + + - name: Restore data-schemas build cache + id: cache-data-schemas + uses: actions/cache@v4 + with: + path: packages/data-schemas/dist + key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }} + + - name: Build data-schemas + if: steps.cache-data-schemas.outputs.cache-hit != 'true' + run: npm run build:data-schemas + + - name: Restore api build cache + id: cache-api + uses: actions/cache@v4 + with: + path: packages/api/dist + key: build-api-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/tsdown.config.mjs', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json') }} + + - name: Build api + if: steps.cache-api.outputs.cache-hit != 'true' + run: npm run build:api + + - name: Restore client-package build cache + id: cache-client-package + uses: actions/cache@v4 + with: + path: packages/client/dist + key: build-client-package-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }} + + - name: Build client-package + if: steps.cache-client-package.outputs.cache-hit != 'true' + run: npm run build:client-package + + - name: Restore client app build cache + id: cache-client-app + uses: actions/cache@v4 + with: + path: client/dist + key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package-lock.json', 'client/src/**', 'client/public/**', 'client/scripts/post-build.cjs', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }} + + - name: Build client app + if: steps.cache-client-app.outputs.cache-hit != 'true' + run: npm run build:client + + - name: Install Playwright runtime dependencies + timeout-minutes: 5 + run: | + google-chrome --version + npx playwright install-deps chrome + + - name: Run five-minute Bombadil exploration + id: bombadil + continue-on-error: true + run: | + set -o pipefail + mkdir -p e2e/.generated + npx playwright test \ + --config=e2e/playwright.config.bombadil.ts \ + --reporter=line,html \ + 2>&1 | tee e2e/.generated/bombadil-ci.log + env: + CI: 'true' + PLAYWRIGHT_HTML_OPEN: 'never' + PLAYWRIGHT_HTML_OUTPUT_DIR: e2e/playwright-report-bombadil + + - name: Upload Bombadil reproduction trace + id: bombadil-reproduction + if: steps.bombadil.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: bombadil-reproduction-${{ github.run_id }}-${{ github.run_attempt }} + path: e2e/.generated/bombadil-output/** + include-hidden-files: true + retention-days: 7 + if-no-files-found: warn + + - name: Upload Bombadil diagnostics + id: bombadil-diagnostics + if: steps.bombadil.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: bombadil-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: | + e2e/.generated/bombadil-ci.log + e2e/playwright-report-bombadil/** + e2e/specs/.test-results/** + include-hidden-files: true + retention-days: 7 + if-no-files-found: warn + + - name: Report non-blocking Bombadil failure + if: steps.bombadil.outcome == 'failure' + run: | + echo "::warning title=Bombadil property violation::The five-minute exploration failed. Download the reproduction and diagnostics artifacts for this run." + { + echo "### Bombadil property exploration" + echo + echo "The exploration failed, but this job does not block merge." + echo + echo "Reproduction: ${{ steps.bombadil-reproduction.outputs.artifact-url }}" + echo + echo "Diagnostics: ${{ steps.bombadil-diagnostics.outputs.artifact-url }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index a44f3797af..ae44060e83 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ src/style - official.css /e2e/benchmarks/.test-results/ /e2e/.generated/ /e2e/playwright-report/ +/e2e/playwright-report-bombadil/ /playwright/.cache/ .DS_Store *.code-workspace diff --git a/e2e/README.md b/e2e/README.md index d0dae2c45a..49afe34305 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -18,6 +18,76 @@ CI runs the complete mock suite in both stream modes. Each mode is split across npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4 ``` +## Property-based browser testing + +Bombadil explores randomized sequences across the core chat loop, message branches, +parallel multi-conversation responses, model changes, reloads, and sidebar conversation +lifecycle operations: + +```sh +npm run e2e:bombadil +``` + +Set `BOMBADIL_TIME_LIMIT` for longer local or scheduled runs. Failures leave a +reproducible trace under `e2e/.generated/bombadil-output`; rerun it with: + +```sh +BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run +``` + +Reproducing a real violation is expected to fail the Playwright test. Before a +new run overwrites the active output, the harness archives it under +`e2e/.generated/bombadil-history/`. Reproduction can diverge when streaming +timing changes; Bombadil reports that explicitly. + +The harness uses the credential-free mock-LLM profile, so exploration never sends +billable provider requests. + +CI runs the broad property exploration for five minutes in the non-blocking +`Bombadil Property Exploration` workflow. If a property fails, download the +`bombadil-reproduction-*` artifact into +`e2e/.generated/bombadil-output/`, then reproduce it locally: + +```sh +BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run +``` + +The accompanying `bombadil-diagnostics-*` artifact contains the captured CI log, +Playwright HTML report, and Playwright test results. A Bombadil failure produces +a workflow warning but does not block merge. + +The default instruments inline JavaScript only because instrumenting LibreChat's +full Vite bundle can exceed Bombadil's driver timeout during stateful runs. Set +`BOMBADIL_INSTRUMENT_JAVASCRIPT=files,inline` for shorter coverage-guided +experiments. + +The branch reload, fork submission, model/conversation, HITL pause/resume, and +mid-run steering lifecycle properties can be run independently: + +```sh +npm run e2e:bombadil:branch-reload +npm run e2e:bombadil:fork-lifecycle +npm run e2e:bombadil:model-lifecycle +npm run e2e:bombadil:hitl +npm run e2e:bombadil:steering +``` + +These focused commands are diagnostic properties: they exit nonzero when they +reproduce a product invariant violation. Reproduce a focused trace with its +matching `:run` script and output directory, for example: + +```sh +BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output-hitl npm run e2e:bombadil:hitl:run +``` + +HITL drives a real `ask_user_question` checkpoint through the answer/resume +controller, reloads while the question is paused, answers it once, and reloads +the completed conversation. Steering submits an in-flight steer during a slow +MCP-backed run, checks that it moves exactly once from the composer anchor into +the response at the tool boundary, and reloads the applied state. The model +lifecycle property is the passing control. The branch reload and fork +properties preserve their minimal failing traces. + ## Recording Tests Use Playwright codegen when you want to turn an exploratory browser session into a draft test: diff --git a/e2e/bombadil/branch-reload.specification.ts b/e2e/bombadil/branch-reload.specification.ts new file mode 100644 index 0000000000..b373796f8a --- /dev/null +++ b/e2e/bombadil/branch-reload.specification.ts @@ -0,0 +1,228 @@ +import { always, eventually, extract } from '@antithesishq/bombadil'; +import { actions } from '@antithesishq/bombadil/browser'; +import type { Action, Point, State } from '@antithesishq/bombadil/browser'; +import { + noConsoleErrors, + noHttpErrorCodes, + noUncaughtExceptions, + noUnhandledPromiseRejections, +} from '@antithesishq/bombadil/browser/defaults/properties'; + +type Target = { + name: string; + point: Point; +}; + +type NavigationStatus = { + current: number; + total: number; +}; + +const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__'; +const ENTER_KEY_CODE = 13; +const BRANCH_PROMPT = 'E2E_REPLY:bombadil-branch-reload'; +const RELOAD_MARKER = 'bombadil-reload-marker'; + +function visiblePoint(state: State, element: Element | null): Point | null { + if (!element) { + return null; + } + const style = state.window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.pointerEvents === 'none' || + rect.width <= 0 || + rect.height <= 0 + ) { + return null; + } + const point = { + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }; + if ( + point.x < 0 || + point.y < 0 || + point.x > state.window.innerWidth || + point.y > state.window.innerHeight + ) { + return null; + } + const hitElement = state.document.elementFromPoint(point.x, point.y); + if (!hitElement || (hitElement !== element && !element.contains(hitElement))) { + return null; + } + return point; +} + +function target(state: State, selector: string, name: string, last = false): Target | null { + const candidates = Array.from(state.document.querySelectorAll(selector)); + const elements = last ? candidates.reverse() : candidates; + for (const element of elements) { + if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') { + continue; + } + const point = visiblePoint(state, element); + if (point) { + return { name, point }; + } + } + return null; +} + +function inputValue(state: State, selector: string): string { + const element = state.document.querySelector(selector) as + | HTMLInputElement + | HTMLTextAreaElement + | null; + return element?.value ?? ''; +} + +function isFocused(state: State, selector: string): boolean { + return state.document.activeElement?.matches(selector) === true; +} + +function clickOrWait(targetValue: Target | null): Action[] { + return targetValue ? [{ Click: targetValue }] : ['Wait']; +} + +function isNamedClick(action: Action | null, name: string): boolean { + return ( + typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name + ); +} + +const ui = extract((state: State) => { + const statuses: NavigationStatus[] = []; + for (const navigation of state.document.querySelectorAll( + 'nav[aria-label="Sibling message navigation"]', + )) { + const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? ''; + const match = text.match(/^(\d+)\s*\/\s*(\d+)$/); + if (match) { + statuses.push({ current: Number(match[1]), total: Number(match[2]) }); + } + } + const encodedReloadMarker = state.window.btoa(RELOAD_MARKER); + const hasSavedReloadMarker = Object.keys(state.window.localStorage).some( + (key) => + key.startsWith('textDraft_') && + state.window.localStorage.getItem(key) === encodedReloadMarker, + ); + + return { + path: state.window.location.pathname, + lastAction: state.lastAction, + messageCount: state.document.querySelectorAll('.message-render').length, + statuses, + hasSavedReloadMarker, + composerValue: inputValue(state, '#prompt-textarea'), + composerFocused: isFocused(state, '#prompt-textarea'), + emailValue: inputValue(state, '#email'), + emailFocused: isFocused(state, '#email'), + passwordValue: inputValue(state, '#password'), + passwordFocused: isFocused(state, '#password'), + isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null, + hasComposer: state.document.querySelector('#prompt-textarea') !== null, + loginEmail: target(state, '#email', 'Login email'), + loginPassword: target(state, '#password', 'Login password'), + loginSubmit: target(state, '[data-testid="login-button"]', 'Login'), + composer: target(state, '#prompt-textarea', 'Message input'), + regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', true), + previousSibling: target( + state, + 'button[aria-label="Previous sibling message"]', + 'Previous sibling message', + true, + ), + }; +}); + +export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections }; + +export const branchReloadActions = actions((): Action[] => { + const state = ui.current; + + if (state.path === '/login') { + if (!state.emailFocused && state.emailValue === '') { + return clickOrWait(state.loginEmail); + } + if (state.emailFocused && state.emailValue === '') { + return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }]; + } + if (!state.passwordFocused && state.passwordValue === '') { + return clickOrWait(state.loginPassword); + } + if (state.passwordFocused && state.passwordValue === '') { + return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }]; + } + return clickOrWait(state.loginSubmit); + } + + if (state.isSubmitting || !state.hasComposer) { + return ['Wait']; + } + + const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new'; + if (isPersistedConversation && state.messageCount === 0) { + return ['Wait']; + } + + if (state.messageCount === 0) { + if (state.composerValue === '') { + if (state.composerFocused) { + return [{ TypeText: { text: BRANCH_PROMPT, delayMillis: 0 } }]; + } + return clickOrWait(state.composer); + } + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + + const status = state.statuses[state.statuses.length - 1]; + if (state.lastAction === 'Reload') { + return ['Wait']; + } + if (!status) { + if (isNamedClick(state.lastAction, 'Regenerate')) { + return ['Wait']; + } + return clickOrWait(state.regenerate); + } + if (status.total === 2 && status.current === 2) { + if (state.hasSavedReloadMarker) { + return ['Wait']; + } + return clickOrWait(state.previousSibling); + } + if (status.total === 2 && status.current === 1) { + if (state.composerValue !== RELOAD_MARKER) { + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ TypeText: { text: RELOAD_MARKER, delayMillis: 0 } }]; + } + if (!state.hasSavedReloadMarker) { + return ['Wait']; + } + return ['Reload']; + } + return ['Wait']; +}); + +export const siblingBranchEventuallyExists = eventually(() => + ui.current.statuses.some(({ total }) => total === 2), +).within(45, 'seconds'); + +export const selectedSiblingSurvivesReload = always( + () => + !ui.current.hasSavedReloadMarker || + ui.current.messageCount === 0 || + !ui.current.statuses.some(({ total }) => total === 2) || + ui.current.statuses.some(({ current, total }) => current === 1 && total === 2), +); diff --git a/e2e/bombadil/fork-lifecycle.specification.ts b/e2e/bombadil/fork-lifecycle.specification.ts new file mode 100644 index 0000000000..1930ddfa6f --- /dev/null +++ b/e2e/bombadil/fork-lifecycle.specification.ts @@ -0,0 +1,202 @@ +import { always, eventually, extract, next, now } from '@antithesishq/bombadil'; +import { actions } from '@antithesishq/bombadil/browser'; +import type { Action, Point, State } from '@antithesishq/bombadil/browser'; +import { + noConsoleErrors, + noHttpErrorCodes, + noUncaughtExceptions, + noUnhandledPromiseRejections, +} from '@antithesishq/bombadil/browser/defaults/properties'; + +type Target = { + name: string; + point: Point; +}; + +const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__'; +const ENTER_KEY_CODE = 13; +const FORK_PROMPT = 'E2E_REPLY:bombadil-fork-lifecycle'; + +function visiblePoint(state: State, element: Element | null): Point | null { + if (!element) { + return null; + } + const style = state.window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.pointerEvents === 'none' || + rect.width <= 0 || + rect.height <= 0 + ) { + return null; + } + const point = { + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }; + if ( + point.x < 0 || + point.y < 0 || + point.x > state.window.innerWidth || + point.y > state.window.innerHeight + ) { + return null; + } + const hitElement = state.document.elementFromPoint(point.x, point.y); + if (!hitElement || (hitElement !== element && !element.contains(hitElement))) { + return null; + } + return point; +} + +function target( + state: State, + selector: string, + name: string, + text?: string, + last = false, +): Target | null { + const candidates = Array.from(state.document.querySelectorAll(selector)).filter( + (element) => text == null || element.textContent?.trim() === text, + ); + const elements = last ? candidates.reverse() : candidates; + for (const element of elements) { + if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') { + continue; + } + const point = visiblePoint(state, element); + if (point) { + return { name, point }; + } + } + return null; +} + +function inputValue(state: State, selector: string): string { + const element = state.document.querySelector(selector) as + | HTMLInputElement + | HTMLTextAreaElement + | null; + return element?.value ?? ''; +} + +function isFocused(state: State, selector: string): boolean { + return state.document.activeElement?.matches(selector) === true; +} + +function clickOrWait(targetValue: Target | null): Action[] { + return targetValue ? [{ Click: targetValue }] : ['Wait']; +} + +function isNamedClick(action: Action | null, name: string): boolean { + return ( + typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name + ); +} + +const ui = extract((state: State) => ({ + path: state.window.location.pathname, + lastAction: state.lastAction, + bodyText: state.document.body.textContent ?? '', + messageCount: state.document.querySelectorAll('.message-render').length, + composerValue: inputValue(state, '#prompt-textarea'), + composerFocused: isFocused(state, '#prompt-textarea'), + emailValue: inputValue(state, '#email'), + emailFocused: isFocused(state, '#email'), + passwordValue: inputValue(state, '#password'), + passwordFocused: isFocused(state, '#password'), + isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null, + hasComposer: state.document.querySelector('#prompt-textarea') !== null, + loginEmail: target(state, '#email', 'Login email'), + loginPassword: target(state, '#password', 'Login password'), + loginSubmit: target(state, '[data-testid="login-button"]', 'Login'), + composer: target(state, '#prompt-textarea', 'Message input'), + forkMenu: target(state, 'button[aria-label="Open Fork Menu"]', 'Open fork menu', undefined, true), + forkVisible: target(state, 'button', 'Fork visible messages', 'Visible messages only', true), +})); + +export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections }; + +export const forkLifecycleActions = actions((): Action[] => { + const state = ui.current; + + if (state.path === '/login') { + if (!state.emailFocused && state.emailValue === '') { + return clickOrWait(state.loginEmail); + } + if (state.emailFocused && state.emailValue === '') { + return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }]; + } + if (!state.passwordFocused && state.passwordValue === '') { + return clickOrWait(state.loginPassword); + } + if (state.passwordFocused && state.passwordValue === '') { + return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }]; + } + return clickOrWait(state.loginSubmit); + } + + if (state.isSubmitting || !state.hasComposer) { + return ['Wait']; + } + + const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new'; + if (isPersistedConversation && state.messageCount === 0) { + return ['Wait']; + } + + if (state.messageCount === 0) { + if (state.composerValue === '') { + if (state.composerFocused) { + return [{ TypeText: { text: FORK_PROMPT, delayMillis: 0 } }]; + } + return clickOrWait(state.composer); + } + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + + if (state.forkVisible) { + return clickOrWait(state.forkVisible); + } + return clickOrWait(state.forkMenu); +}); + +export const forkSetupEventuallyReachesChoice = eventually( + () => ui.current.forkVisible !== null, +).within(45, 'seconds'); + +export const forkChoiceBecomesUnavailableAfterSubmission = always(() => { + const choiceWasVisible = ui.current.forkVisible !== null; + return next( + now( + () => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'), + ).implies(now(() => ui.current.forkVisible === null)), + ); +}); + +export const forkSubmissionEventuallyNavigates = always(() => { + const originalPath = ui.current.path; + const choiceWasVisible = ui.current.forkVisible !== null; + return next( + now( + () => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'), + ).implies( + eventually( + () => + ui.current.path.startsWith('/c/') && + ui.current.path !== '/c/new' && + ui.current.path !== originalPath, + ).within(30, 'seconds'), + ), + ); +}); + +export const forkRateLimitIsNeverReached = always( + () => !ui.current.bodyText.includes('Too many fork requests. Please try again later'), +); diff --git a/e2e/bombadil/harness.spec.ts b/e2e/bombadil/harness.spec.ts new file mode 100644 index 0000000000..8c5bc53d75 --- /dev/null +++ b/e2e/bombadil/harness.spec.ts @@ -0,0 +1,176 @@ +import { spawn } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import { expect, test } from '@playwright/test'; +import { getE2EBaseURL } from '../setup/env'; +import { getE2EUser } from '../setup/user'; + +const rootPath = path.resolve(__dirname, '../..'); +const specificationPath = path.resolve( + __dirname, + process.env.BOMBADIL_SPECIFICATION ?? 'specification.ts', +); +const specificationStem = path.basename(specificationPath, '.ts').replace(/\.specification$/, ''); +const defaultOutputPath = path.resolve( + rootPath, + specificationStem === 'specification' + ? 'e2e/.generated/bombadil-output' + : `e2e/.generated/bombadil-output-${specificationStem}`, +); +const defaultBinaryPath = path.resolve(rootPath, 'node_modules/.bin/bombadil'); +const MAX_CAPTURED_OUTPUT = 100_000; +const DEFAULT_REPRODUCTION_TIMEOUT_MS = 30 * 60 * 1_000; +const LOGIN_EMAIL_PLACEHOLDER = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD_PLACEHOLDER = '__BOMBADIL_E2E_USER_PASSWORD__'; + +function durationMillis(value: string): number | null { + const match = value.match(/^(\d+)(s|m|h|d)$/); + if (!match) { + return null; + } + const factors = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }; + return Number(match[1]) * factors[match[2] as keyof typeof factors]; +} + +function positiveTimeoutMillis(value: string, variableName: string): number { + const timeout = Number(value); + if (!Number.isFinite(timeout) || timeout <= 0) { + throw new Error( + `${variableName} must be a positive number of milliseconds; received "${value}".`, + ); + } + return timeout; +} + +function archiveExistingOutput(outputPath: string): void { + if (!fs.existsSync(outputPath)) { + return; + } + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const archivePath = path.resolve( + rootPath, + 'e2e/.generated/bombadil-history', + `${path.basename(outputPath)}-${timestamp}`, + ); + fs.mkdirSync(path.dirname(archivePath), { recursive: true }); + fs.cpSync(outputPath, archivePath, { recursive: true }); +} + +function materializeSpecification(): string { + const user = getE2EUser(); + const generatedPath = path.resolve( + rootPath, + 'e2e/.generated/bombadil-specifications', + path.basename(specificationPath), + ); + const replaceStringLiteral = (source: string, placeholder: string, value: string) => + source + .replaceAll(`'${placeholder}'`, JSON.stringify(value)) + .replaceAll(`"${placeholder}"`, JSON.stringify(value)); + let source = fs.readFileSync(specificationPath, 'utf8'); + source = replaceStringLiteral(source, LOGIN_EMAIL_PLACEHOLDER, user.email); + source = replaceStringLiteral(source, LOGIN_PASSWORD_PLACEHOLDER, user.password); + fs.mkdirSync(path.dirname(generatedPath), { recursive: true }); + fs.writeFileSync(generatedPath, source); + return generatedPath; +} + +function runBombadil( + outputPath: string, + runtimeSpecificationPath: string, + childTimeoutMillis: number, +): Promise<{ code: number | null; output: string }> { + const binaryPath = process.env.BOMBADIL_BIN ?? defaultBinaryPath; + const reproducePath = process.env.BOMBADIL_REPRODUCE; + const timeLimit = process.env.BOMBADIL_TIME_LIMIT ?? '90s'; + const args = [ + 'browser', + 'test', + '--headless', + '--output-path', + outputPath, + '--output-path-overwrite', + '--instrument-javascript', + // Instrumenting LibreChat's full Vite bundle makes Bombadil's driver time out + // under longer stateful runs. Full `files,inline` coverage remains opt-in. + process.env.BOMBADIL_INSTRUMENT_JAVASCRIPT ?? 'inline', + ]; + + if (reproducePath) { + args.push('--reproduce', reproducePath); + } else { + args.push('--exit-on-violation', '--time-limit', timeLimit); + } + + args.push(getE2EBaseURL(), runtimeSpecificationPath); + + return new Promise((resolve, reject) => { + const child = spawn(binaryPath, args, { + cwd: rootPath, + env: { + ...process.env, + RUST_LOG: process.env.BOMBADIL_RUST_LOG ?? 'error', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + let timedOut = false; + let forceKillTimer: NodeJS.Timeout | undefined; + const childTimer = setTimeout(() => { + timedOut = true; + output = `${output}\nBombadil exceeded the ${childTimeoutMillis}ms child-process timeout.`; + child.kill('SIGTERM'); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 5_000); + }, childTimeoutMillis); + const append = (chunk: Buffer) => { + const text = chunk.toString(); + output = `${output}${text}`.slice(-MAX_CAPTURED_OUTPUT); + process.stdout.write(text); + }; + + child.stdout.on('data', append); + child.stderr.on('data', append); + child.once('error', (error) => { + clearTimeout(childTimer); + if (forceKillTimer) { + clearTimeout(forceKillTimer); + } + reject(error); + }); + child.once('exit', (code) => { + clearTimeout(childTimer); + if (forceKillTimer) { + clearTimeout(forceKillTimer); + } + resolve({ code: timedOut ? null : code, output }); + }); + }); +} + +test('Bombadil explores core, branching, multi-conversation, and lifecycle flows', async () => { + const timeLimitMillis = durationMillis(process.env.BOMBADIL_TIME_LIMIT ?? '90s') ?? 90_000; + const defaultHarnessTimeout = process.env.BOMBADIL_REPRODUCE + ? DEFAULT_REPRODUCTION_TIMEOUT_MS + : timeLimitMillis + 120_000; + const harnessTimeout = process.env.BOMBADIL_HARNESS_TIMEOUT_MS + ? positiveTimeoutMillis(process.env.BOMBADIL_HARNESS_TIMEOUT_MS, 'BOMBADIL_HARNESS_TIMEOUT_MS') + : defaultHarnessTimeout; + test.setTimeout(harnessTimeout); + const outputPath = process.env.BOMBADIL_REPRODUCE + ? path.resolve(rootPath, 'e2e/.generated/bombadil-reproduction') + : defaultOutputPath; + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + archiveExistingOutput(outputPath); + const runtimeSpecificationPath = materializeSpecification(); + + const result = await runBombadil( + outputPath, + runtimeSpecificationPath, + Math.max(harnessTimeout - 10_000, 10_000), + ); + + expect( + result.code, + `Bombadil exited with ${result.code}. Inspect or reproduce the trace at ${outputPath}.\n${result.output}`, + ).toBe(0); +}); diff --git a/e2e/bombadil/hitl-lifecycle.specification.ts b/e2e/bombadil/hitl-lifecycle.specification.ts new file mode 100644 index 0000000000..0c00517559 --- /dev/null +++ b/e2e/bombadil/hitl-lifecycle.specification.ts @@ -0,0 +1,248 @@ +import { always, eventually, extract, now } from '@antithesishq/bombadil'; +import { actions } from '@antithesishq/bombadil/browser'; +import type { Action, Point, State } from '@antithesishq/bombadil/browser'; +import { + noConsoleErrors, + noHttpErrorCodes, + noUncaughtExceptions, + noUnhandledPromiseRejections, +} from '@antithesishq/bombadil/browser/defaults/properties'; + +type Target = { + name: string; + point: Point; +}; + +const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__'; +const ENTER_KEY_CODE = 13; +const HITL_MODEL_SPEC = 'E2E HITL'; +const HITL_LABEL = 'bombadil-hitl'; +const HITL_PROMPT = `E2E_ASK_USER_QUESTION:${HITL_LABEL}`; +const HITL_QUESTION = `Which environment should Bombadil use for ${HITL_LABEL}?`; +const HITL_OPTION = 'Staging'; +const FINAL_REPLY = 'E2E mock reply: pong'; +const COMPLETED_ANSWER = 'You answered: Staging'; +let reloadIssued = false; +let pausedReloadIssued = false; + +function visiblePoint(state: State, element: Element | null): Point | null { + if (!element) { + return null; + } + const style = state.window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.pointerEvents === 'none' || + rect.width <= 0 || + rect.height <= 0 + ) { + return null; + } + const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + const hitElement = state.document.elementFromPoint(point.x, point.y); + if ( + point.x < 0 || + point.y < 0 || + point.x > state.window.innerWidth || + point.y > state.window.innerHeight || + !hitElement || + (hitElement !== element && !element.contains(hitElement)) + ) { + return null; + } + return point; +} + +function target( + state: State, + selector: string, + name: string, + text?: string, + containsText = false, +): Target | null { + for (const element of state.document.querySelectorAll(selector)) { + const content = element.textContent?.trim() ?? ''; + if (text != null && (containsText ? !content.includes(text) : content !== text)) { + continue; + } + if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') { + continue; + } + const point = visiblePoint(state, element); + if (point) { + return { name, point }; + } + } + return null; +} + +function visibleTextCount( + state: State, + selector: string, + text: string, + containsText = false, +): number { + return Array.from(state.document.querySelectorAll(selector)).filter((element) => { + const content = element.textContent?.trim() ?? ''; + return ( + (containsText ? content.includes(text) : content === text) && + visiblePoint(state, element) !== null + ); + }).length; +} + +function inputValue(state: State, selector: string): string { + return ( + state.document.querySelector(selector)?.value ?? '' + ); +} + +function isFocused(state: State, selector: string): boolean { + return state.document.activeElement?.matches(selector) === true; +} + +function clickOrWait(targetValue: Target | null): Action[] { + return targetValue ? [{ Click: targetValue }] : ['Wait']; +} + +const ui = extract((state: State) => { + const messageElements = Array.from(state.document.querySelectorAll('.message-render')); + const messageText = messageElements.map((element) => element.textContent ?? '').join('\n'); + const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]'); + return { + path: state.window.location.pathname, + lastAction: state.lastAction, + messageCount: messageElements.length, + messageText, + modelLabel: modelTrigger?.textContent?.trim() ?? '', + composerValue: inputValue(state, '#prompt-textarea'), + composerFocused: isFocused(state, '#prompt-textarea'), + emailValue: inputValue(state, '#email'), + emailFocused: isFocused(state, '#email'), + passwordValue: inputValue(state, '#password'), + passwordFocused: isFocused(state, '#password'), + questionCount: visibleTextCount(state, 'p', HITL_QUESTION), + answerOptionCount: visibleTextCount(state, 'button', HITL_OPTION, true), + finalReplyCount: messageElements.filter((element) => + (element.textContent ?? '').includes(FINAL_REPLY), + ).length, + completedAnswerCount: messageElements.filter((element) => + (element.textContent ?? '').includes(COMPLETED_ANSWER), + ).length, + isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null, + hasComposer: state.document.querySelector('#prompt-textarea') !== null, + loginEmail: target(state, '#email', 'Login email'), + loginPassword: target(state, '#password', 'Login password'), + loginSubmit: target(state, '[data-testid="login-button"]', 'Login'), + composer: target(state, '#prompt-textarea', 'Message input'), + modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'), + hitlModelSpec: target(state, '[role="option"]', HITL_MODEL_SPEC, HITL_MODEL_SPEC), + stagingOption: target(state, 'button', 'Answer Staging', HITL_OPTION, true), + }; +}); + +export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections }; + +export const hitlLifecycleActions = actions((): Action[] => { + const state = ui.current; + + if (state.path === '/login') { + reloadIssued = false; + pausedReloadIssued = false; + if (!state.emailFocused && state.emailValue === '') { + return clickOrWait(state.loginEmail); + } + if (state.emailFocused && state.emailValue === '') { + return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }]; + } + if (!state.passwordFocused && state.passwordValue === '') { + return clickOrWait(state.loginPassword); + } + if (state.passwordFocused && state.passwordValue === '') { + return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }]; + } + return clickOrWait(state.loginSubmit); + } + + if (state.stagingOption) { + if (!pausedReloadIssued) { + pausedReloadIssued = true; + return ['Reload']; + } + return clickOrWait(state.stagingOption); + } + + if (state.isSubmitting || !state.hasComposer) { + return ['Wait']; + } + + if (state.finalReplyCount === 1) { + if (!reloadIssued) { + reloadIssued = true; + return ['Reload']; + } + return ['Wait']; + } + + const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new'; + if (isPersistedConversation && state.messageCount === 0) { + return ['Wait']; + } + + if (state.messageCount === 0) { + if (state.modelLabel !== HITL_MODEL_SPEC) { + return state.hitlModelSpec + ? clickOrWait(state.hitlModelSpec) + : clickOrWait(state.modelTrigger); + } + if (state.composerValue === '') { + return state.composerFocused + ? [{ TypeText: { text: HITL_PROMPT, delayMillis: 0 } }] + : clickOrWait(state.composer); + } + return state.composerFocused + ? [{ PressKey: { code: ENTER_KEY_CODE } }] + : clickOrWait(state.composer); + } + + return ['Wait']; +}); + +/** The run must reach a real, answerable ask_user_question pause, including after reload. */ +export const hitlQuestionEventuallyPauses = eventually( + () => ui.current.questionCount === 1 && ui.current.answerOptionCount === 1, +).within(25, 'seconds'); + +/** Answering resumes the checkpointed run and produces one terminal reply. */ +export const hitlAnswerEventuallyResumes = eventually( + () => + ui.current.finalReplyCount === 1 && + ui.current.completedAnswerCount === 1 && + ui.current.answerOptionCount === 0, +).within(40, 'seconds'); + +/** Duplicate cards or duplicate resume completions indicate a broken pause lifecycle. */ +export const hitlPauseAndResumeStaySingular = always( + () => + ui.current.questionCount <= 1 && + ui.current.answerOptionCount <= 1 && + ui.current.finalReplyCount <= 1 && + ui.current.completedAnswerCount <= 1 && + ui.current.messageCount <= 2, +); + +/** After reload, the question remains an audit record without becoming answerable again. */ +export const answeredHitlStateSurvivesReload = always(() => + now(() => ui.current.lastAction === 'Reload').implies( + eventually( + () => + ui.current.finalReplyCount === 1 && + ui.current.completedAnswerCount === 1 && + ui.current.questionCount === 1 && + ui.current.answerOptionCount === 0, + ).within(20, 'seconds'), + ), +); diff --git a/e2e/bombadil/model-lifecycle.specification.ts b/e2e/bombadil/model-lifecycle.specification.ts new file mode 100644 index 0000000000..21e2540fb9 --- /dev/null +++ b/e2e/bombadil/model-lifecycle.specification.ts @@ -0,0 +1,211 @@ +import { always, eventually, extract, now } from '@antithesishq/bombadil'; +import { actions } from '@antithesishq/bombadil/browser'; +import type { Action, Point, State } from '@antithesishq/bombadil/browser'; +import { + noConsoleErrors, + noHttpErrorCodes, + noUncaughtExceptions, + noUnhandledPromiseRejections, +} from '@antithesishq/bombadil/browser/defaults/properties'; + +type Target = { + name: string; + point: Point; +}; + +const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__'; +const ENTER_KEY_CODE = 13; +const TARGET_MODEL_SPEC = 'E2E Starters'; +const INITIAL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-initial'; +const SELECTED_MODEL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-selected'; +let reloadIssued = false; + +function visiblePoint(state: State, element: Element | null): Point | null { + if (!element) { + return null; + } + const style = state.window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.pointerEvents === 'none' || + rect.width <= 0 || + rect.height <= 0 + ) { + return null; + } + const point = { + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }; + if ( + point.x < 0 || + point.y < 0 || + point.x > state.window.innerWidth || + point.y > state.window.innerHeight + ) { + return null; + } + const hitElement = state.document.elementFromPoint(point.x, point.y); + if (!hitElement || (hitElement !== element && !element.contains(hitElement))) { + return null; + } + return point; +} + +function target(state: State, selector: string, name: string, text?: string): Target | null { + for (const element of state.document.querySelectorAll(selector)) { + if (text != null && element.textContent?.trim() !== text) { + continue; + } + if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') { + continue; + } + const point = visiblePoint(state, element); + if (point) { + return { name, point }; + } + } + return null; +} + +function inputValue(state: State, selector: string): string { + const element = state.document.querySelector(selector) as + | HTMLInputElement + | HTMLTextAreaElement + | null; + return element?.value ?? ''; +} + +function isFocused(state: State, selector: string): boolean { + return state.document.activeElement?.matches(selector) === true; +} + +function clickOrWait(targetValue: Target | null): Action[] { + return targetValue ? [{ Click: targetValue }] : ['Wait']; +} + +const ui = extract((state: State) => { + const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]'); + return { + path: state.window.location.pathname, + lastAction: state.lastAction, + messageCount: state.document.querySelectorAll('.message-render').length, + composerValue: inputValue(state, '#prompt-textarea'), + composerFocused: isFocused(state, '#prompt-textarea'), + emailValue: inputValue(state, '#email'), + emailFocused: isFocused(state, '#email'), + passwordValue: inputValue(state, '#password'), + passwordFocused: isFocused(state, '#password'), + modelLabel: modelTrigger?.textContent?.trim() ?? '', + isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null, + hasComposer: state.document.querySelector('#prompt-textarea') !== null, + loginEmail: target(state, '#email', 'Login email'), + loginPassword: target(state, '#password', 'Login password'), + loginSubmit: target(state, '[data-testid="login-button"]', 'Login'), + composer: target(state, '#prompt-textarea', 'Message input'), + modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'), + starterSpec: target(state, '[role="option"]', TARGET_MODEL_SPEC, TARGET_MODEL_SPEC), + }; +}); + +export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections }; + +export const modelLifecycleActions = actions((): Action[] => { + const state = ui.current; + + if (state.path === '/login') { + reloadIssued = false; + if (!state.emailFocused && state.emailValue === '') { + return clickOrWait(state.loginEmail); + } + if (state.emailFocused && state.emailValue === '') { + return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }]; + } + if (!state.passwordFocused && state.passwordValue === '') { + return clickOrWait(state.loginPassword); + } + if (state.passwordFocused && state.passwordValue === '') { + return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }]; + } + return clickOrWait(state.loginSubmit); + } + + if (state.isSubmitting || !state.hasComposer) { + return ['Wait']; + } + + const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new'; + if (isPersistedConversation && state.messageCount === 0) { + return ['Wait']; + } + + if (state.messageCount === 0) { + if (state.composerValue === '') { + if (state.composerFocused) { + return [{ TypeText: { text: INITIAL_PROMPT, delayMillis: 0 } }]; + } + return clickOrWait(state.composer); + } + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + + if (state.messageCount === 2) { + if (state.modelLabel !== TARGET_MODEL_SPEC) { + if (state.starterSpec) { + return clickOrWait(state.starterSpec); + } + return clickOrWait(state.modelTrigger); + } + if (state.composerValue === '') { + if (state.composerFocused) { + return [{ TypeText: { text: SELECTED_MODEL_PROMPT, delayMillis: 0 } }]; + } + return clickOrWait(state.composer); + } + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + + if (state.messageCount >= 4 && state.modelLabel === TARGET_MODEL_SPEC) { + if (!reloadIssued) { + reloadIssued = true; + return ['Reload']; + } + return ['Wait']; + } + + return ['Wait']; +}); + +/** + * A model choice can remain local until it is used for a submission. Once a turn + * has been sent with that model, both the selection and message history must be + * server-backed and recoverable after reload. + */ +export const submittedModelAndMessagesSurviveReload = always( + () => + ui.current.messageCount < 4 || + ui.current.modelLabel === '' || + ui.current.modelLabel === TARGET_MODEL_SPEC, +); + +export const selectedModelExchangeEventuallyCommits = eventually( + () => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC, +).within(25, 'seconds'); + +/** The committed model selection and all four turns must rehydrate after reload. */ +export const selectedModelExchangeSurvivesReload = always(() => + now(() => ui.current.lastAction === 'Reload').implies( + eventually( + () => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC, + ).within(20, 'seconds'), + ), +); diff --git a/e2e/bombadil/specification.ts b/e2e/bombadil/specification.ts new file mode 100644 index 0000000000..7b27eb9822 --- /dev/null +++ b/e2e/bombadil/specification.ts @@ -0,0 +1,570 @@ +import { always, eventually, extract, from, integers, next, now } from '@antithesishq/bombadil'; +import { actions } from '@antithesishq/bombadil/browser'; +import type { Action, Point, State } from '@antithesishq/bombadil/browser'; +import { + noConsoleErrors, + noHttpErrorCodes, + noUncaughtExceptions, + noUnhandledPromiseRejections, +} from '@antithesishq/bombadil/browser/defaults/properties'; + +type Target = { + name: string; + point: Point; +}; + +type NavigationStatus = { + current: number; + total: number; + previousDisabled: boolean; + nextDisabled: boolean; +}; + +const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__'; +const ENTER_KEY_CODE = 13; +const RENAME_MARKER_PREFIX = ' — Bombadil:'; +const MAX_CONVERSATION_FINGERPRINT_LENGTH = 80; +const PROMPT_VARIANTS = from([ + { name: 'short', suffix: '' }, + { name: 'unicode-雪-🙂', suffix: '' }, + { name: 'spaces', suffix: ' with internal spaces' }, + { name: 'markdown', suffix: ' [x](y)' }, + { name: 'long', suffix: `-${'x'.repeat(256)}` }, +]); +const PROMPT_NONCES = integers().min(0).max(2_147_483_647); + +function generatePrompt(): string { + const variant = PROMPT_VARIANTS.generate(); + return `E2E_REPLY:bombadil-${variant.name}-${PROMPT_NONCES.generate()}${variant.suffix}`; +} + +function promptMarker(text: string): string { + return text.match(/E2E_REPLY:[^\s]+/)?.[0] ?? ''; +} + +function conversationFingerprint(text: string): string { + return promptMarker(text).slice(0, MAX_CONVERSATION_FINGERPRINT_LENGTH); +} + +function expectedReply(text: string): string { + const marker = promptMarker(text); + return marker === '' ? '' : `E2E reply ${marker.slice('E2E_REPLY:'.length)}`; +} + +function visiblePoint(state: State, element: Element | null): Point | null { + if (!element) { + return null; + } + const style = state.window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.pointerEvents === 'none' || + rect.width <= 0 || + rect.height <= 0 + ) { + return null; + } + const point = { + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }; + if ( + point.x < 0 || + point.y < 0 || + point.x > state.window.innerWidth || + point.y > state.window.innerHeight + ) { + return null; + } + const hitElement = state.document.elementFromPoint(point.x, point.y); + if (!hitElement || (hitElement !== element && !element.contains(hitElement))) { + return null; + } + return point; +} + +function target( + state: State, + selector: string, + name: string, + text?: string, + last = false, +): Target | null { + const candidates = Array.from(state.document.querySelectorAll(selector)).filter((element) => { + if (text == null) { + return true; + } + return element.textContent?.trim() === text; + }); + const elements = last ? candidates.reverse() : candidates; + for (const element of elements) { + if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') { + continue; + } + const point = visiblePoint(state, element); + if (point) { + return { name, point }; + } + } + return null; +} + +function conversationTargets(state: State): Target[] { + return Array.from(state.document.querySelectorAll('[data-testid="convo-item"]')).flatMap( + (element, index) => { + if (element.querySelector('[aria-current="page"]')) { + return []; + } + const point = visiblePoint(state, element); + return point ? [{ name: `Open conversation ${index + 1}`, point }] : []; + }, + ); +} + +function clickedConversationIndex(lastAction: unknown): number | null { + if (typeof lastAction !== 'object' || lastAction === null || !('Click' in lastAction)) { + return null; + } + const click = lastAction.Click; + if (typeof click !== 'object' || click === null || !('name' in click)) { + return null; + } + const match = String(click.name).match(/^Open conversation (\d+)$/); + return match ? Number(match[1]) : null; +} + +function isPersistedConversation(pathname: string): boolean { + return pathname.startsWith('/c/') && pathname !== '/c/new'; +} + +function inputValue(state: State, selector: string): string { + const element = state.document.querySelector(selector) as + | HTMLInputElement + | HTMLTextAreaElement + | null; + return element?.value ?? ''; +} + +function isFocused(state: State, selector: string): boolean { + return state.document.activeElement?.matches(selector) === true; +} + +function clickAction(targetValue: Target | null): Action[] { + return targetValue ? [{ Click: targetValue }] : []; +} + +function clickOrWait(targetValue: Target | null): Action[] { + const clicks = clickAction(targetValue); + return clicks.length > 0 ? clicks : ['Wait']; +} + +const ui = extract((state: State) => { + const statuses: NavigationStatus[] = []; + for (const navigation of state.document.querySelectorAll( + 'nav[aria-label="Sibling message navigation"]', + )) { + const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? ''; + const match = text.match(/^(\d+)\s*\/\s*(\d+)$/); + const previous = navigation.querySelector( + 'button[aria-label="Previous sibling message"]', + ) as HTMLButtonElement | null; + const nextButton = navigation.querySelector( + 'button[aria-label="Next sibling message"]', + ) as HTMLButtonElement | null; + if (match && previous && nextButton) { + statuses.push({ + current: Number(match[1]), + total: Number(match[2]), + previousDisabled: previous.disabled || previous.getAttribute('aria-disabled') === 'true', + nextDisabled: nextButton.disabled || nextButton.getAttribute('aria-disabled') === 'true', + }); + } + } + + const messageIds = Array.from(state.document.querySelectorAll('.message-render')) + .map((element) => element.id) + .filter(Boolean); + const messageText = Array.from(state.document.querySelectorAll('.message-render')) + .map((element) => element.textContent ?? '') + .join('\n'); + const parallelColumnCounts = Array.from( + state.document.querySelectorAll('.sibling-content-group'), + ).map((group) => group.children.length); + const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]'); + const conversationElements = Array.from( + state.document.querySelectorAll('[data-testid="convo-item"]'), + ); + const activeConversationIndexes = conversationElements.flatMap((element, index) => + element.querySelector('[aria-current="page"]') ? [index + 1] : [], + ); + const activeConversation = conversationElements.find((element) => + element.querySelector('[aria-current="page"]'), + ); + + return { + path: state.window.location.pathname, + lastAction: state.lastAction, + messageText, + messageIds, + statuses, + parallelColumnCounts, + composerValue: inputValue(state, '#prompt-textarea'), + composerFocused: isFocused(state, '#prompt-textarea'), + renameValue: inputValue(state, 'input[aria-label="New Conversation Title"]'), + renameFocused: isFocused(state, 'input[aria-label="New Conversation Title"]'), + emailValue: inputValue(state, '#email'), + emailFocused: isFocused(state, '#email'), + passwordValue: inputValue(state, '#password'), + passwordFocused: isFocused(state, '#password'), + modelLabel: modelTrigger?.textContent?.trim() ?? '', + activeConversationIndexes, + activeConversationTitle: activeConversation?.textContent?.trim() ?? '', + modelOptionsOpen: state.document.querySelector('[role="option"]') !== null, + hasAddedConversation: + state.document.querySelector('button[aria-label="Close added conversation"]') !== null, + isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null, + hasComposer: state.document.querySelector('#prompt-textarea') !== null, + loginEmail: target(state, '#email', 'Login email'), + loginPassword: target(state, '#password', 'Login password'), + loginSubmit: target(state, '[data-testid="login-button"]', 'Login'), + modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'), + providerA: target(state, '[role="option"]', 'Mock Provider A', 'Mock Provider A'), + providerB: target(state, '[role="option"]', 'Mock Provider B', 'Mock Provider B'), + modelA: target(state, '[role="option"]', 'mock-model-a', 'mock-model-a'), + modelB: target(state, '[role="option"]', 'mock-model-b', 'mock-model-b'), + starterSpec: target(state, '[role="option"]', 'E2E Starters', 'E2E Starters'), + composer: target(state, '#prompt-textarea', 'Message input'), + addMultiConversation: target( + state, + '[data-testid="add-multi-convo-button"]', + 'Add multi-conversation', + ), + closeAddedConversation: target( + state, + 'button[aria-label="Close added conversation"]', + 'Close added conversation', + ), + regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', undefined, true), + previousSibling: target( + state, + 'button[aria-label="Previous sibling message"]', + 'Previous sibling message', + undefined, + true, + ), + nextSibling: target( + state, + 'button[aria-label="Next sibling message"]', + 'Next sibling message', + undefined, + true, + ), + branchParallel: target( + state, + 'button[aria-label="Create branch from this response"]', + 'Create branch from parallel response', + undefined, + true, + ), + newConversation: target(state, '[data-testid="new-chat-button"]', 'New conversation'), + conversationItems: conversationTargets(state), + conversationMenu: target( + state, + 'button[aria-label="Conversation Menu Options"]', + 'Conversation menu', + ), + renameMenuItem: target(state, '[role="menuitem"]', 'Rename conversation', 'Rename'), + renameInput: target(state, 'input[aria-label="New Conversation Title"]', 'Conversation title'), + renameSave: target(state, 'button[aria-label="Save"]', 'Save conversation title'), + }; +}); + +export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections }; + +export const libreChatActions = actions(() => { + const state = ui.current; + + if (state.path === '/login') { + if (!state.emailFocused && state.emailValue === '') { + return clickOrWait(state.loginEmail); + } + if (state.emailFocused && state.emailValue === '') { + return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }]; + } + if (!state.passwordFocused && state.passwordValue === '') { + return clickOrWait(state.loginPassword); + } + if (state.passwordFocused && state.passwordValue === '') { + return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }]; + } + return clickOrWait(state.loginSubmit); + } + + if (state.renameInput) { + if (!state.renameFocused) { + return clickAction(state.renameInput); + } + if (!state.renameValue.includes(RENAME_MARKER_PREFIX.trim())) { + const fingerprint = conversationFingerprint(state.messageText); + return fingerprint === '' + ? ['Wait'] + : [{ TypeText: { text: `${RENAME_MARKER_PREFIX}${fingerprint}`, delayMillis: 0 } }]; + } + return clickOrWait(state.renameSave); + } + + if (state.isSubmitting) { + return ['Wait']; + } + + const selectorActions: Action[] = ['Wait']; + if (state.providerA) { + selectorActions.push(...clickAction(state.providerA)); + } + if (state.providerB) { + selectorActions.push(...clickAction(state.providerB)); + } + if (state.modelA) { + selectorActions.push(...clickAction(state.modelA)); + } + if (state.modelB) { + selectorActions.push(...clickAction(state.modelB)); + } + if (state.starterSpec) { + selectorActions.push(...clickAction(state.starterSpec)); + } + if (state.modelOptionsOpen || selectorActions.length > 1) { + if (state.modelOptionsOpen) { + // Clicking a visible point outside the popover closes unknown submenu + // states without getting trapped in a keyboard-action retry loop. + selectorActions.push(...clickAction(state.composer)); + } + return selectorActions; + } + + if (state.renameMenuItem) { + return ['Wait', ...clickAction(state.renameMenuItem)]; + } + + const isPersisted = isPersistedConversation(state.path); + if (!state.hasComposer || (isPersisted && state.messageIds.length === 0)) { + return ['Wait']; + } + + const possible: Action[] = ['Wait']; + if (state.modelLabel === '' || state.modelLabel === 'Select a model') { + possible.push(...clickAction(state.modelTrigger)); + return possible; + } + + const composeActions = (): Action[] => { + if (state.composerValue !== '') { + if (state.composerFocused) { + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + return clickOrWait(state.composer); + } + if (state.composerFocused) { + return [{ TypeText: { text: generatePrompt(), delayMillis: 0 } }]; + } + return clickOrWait(state.composer); + }; + + if (state.messageIds.length === 0) { + return composeActions(); + } + + if (state.parallelColumnCounts.length === 0) { + if (!state.hasAddedConversation) { + return clickOrWait(state.addMultiConversation); + } + return composeActions(); + } + + if (state.statuses.length === 0 && state.branchParallel) { + return clickAction(state.branchParallel); + } + if (state.composerValue === '') { + if (state.composerFocused) { + possible.push({ TypeText: { text: generatePrompt(), delayMillis: 0 } }); + } else { + possible.push(...clickAction(state.composer)); + } + } else if (state.composerFocused) { + possible.push({ PressKey: { code: ENTER_KEY_CODE } }); + } + + possible.push(...clickAction(state.modelTrigger)); + possible.push(...clickAction(state.addMultiConversation)); + possible.push(...clickAction(state.closeAddedConversation)); + possible.push(...clickAction(state.regenerate)); + possible.push(...clickAction(state.previousSibling)); + possible.push(...clickAction(state.nextSibling)); + possible.push(...clickAction(state.branchParallel)); + possible.push(...clickAction(state.newConversation)); + possible.push(...state.conversationItems.map((item) => ({ Click: item }) as Action)); + possible.push(...clickAction(state.conversationMenu)); + possible.push(...clickAction(state.renameMenuItem)); + + if (state.path.startsWith('/c/') && state.path !== '/c/new') { + possible.push('Reload'); + } + return possible; +}); + +export const messageIdsRemainUnique = always(() => { + const ids = ui.current.messageIds; + return new Set(ids).size === ids.length; +}); + +export const siblingNavigationRemainsValid = always(() => + ui.current.statuses.every( + ({ current, total, previousDisabled, nextDisabled }) => + total > 1 && + current >= 1 && + current <= total && + previousDisabled === (current === 1) && + nextDisabled === (current === total), + ), +); + +export const multiConversationAlwaysRendersTwoColumns = always(() => + ui.current.parallelColumnCounts.every((count) => count === 2), +); + +export const sidebarNavigationEventuallySelectsTarget = always(() => { + const expectedIndex = clickedConversationIndex(ui.current.lastAction); + return now(() => expectedIndex !== null).implies( + eventually( + () => + expectedIndex !== null && + isPersistedConversation(ui.current.path) && + ui.current.activeConversationIndexes.length === 1 && + ui.current.activeConversationIndexes[0] === expectedIndex && + ui.current.messageIds.length > 0, + ).within(10, 'seconds'), + ); +}); + +export const sidebarEventuallyMatchesRenderedConversation = always(() => + now( + () => + isPersistedConversation(ui.current.path) && + ui.current.hasComposer && + ui.current.messageIds.length > 0, + ).implies( + eventually(() => { + const state = ui.current; + const fingerprint = conversationFingerprint(state.activeConversationTitle); + return ( + isPersistedConversation(state.path) && + state.activeConversationIndexes.length === 1 && + (fingerprint === '' || state.messageText.includes(fingerprint)) + ); + }).within(10, 'seconds'), + ), +); + +export const loginEventuallySucceeds = eventually(() => ui.current.path !== '/login').within( + 30, + 'seconds', +); + +export const anExchangeEventuallyHappens = eventually( + () => ui.current.messageIds.length >= 2, +).within(60, 'seconds'); + +export const multiConversationEventuallyRenders = eventually( + () => ui.current.parallelColumnCounts.length > 0, +).within(75, 'seconds'); + +export const aBranchEventuallyRenders = eventually(() => ui.current.statuses.length > 0).within( + 85, + 'seconds', +); + +export const streamingEventuallyTerminates = always(() => + now(() => ui.current.isSubmitting).implies( + eventually(() => !ui.current.isSubmitting).within(45, 'seconds'), + ), +); + +export const submittedPromptEventuallyAppears = always(() => { + const submittedText = ui.current.composerValue.trim(); + const marker = promptMarker(submittedText); + const eligible = ui.current.composerFocused && submittedText !== ''; + return next( + now( + () => + eligible && + typeof ui.current.lastAction === 'object' && + ui.current.lastAction !== null && + 'PressKey' in ui.current.lastAction && + ui.current.lastAction.PressKey.code === ENTER_KEY_CODE, + ).implies( + eventually(() => marker !== '' && ui.current.messageText.includes(marker)).within( + 30, + 'seconds', + ), + ), + ); +}); + +export const assistantReplyEventuallyAppears = always(() => { + const replyText = expectedReply(ui.current.composerValue.trim()); + const eligible = ui.current.composerFocused && replyText !== ''; + return next( + now( + () => + eligible && + typeof ui.current.lastAction === 'object' && + ui.current.lastAction !== null && + 'PressKey' in ui.current.lastAction && + ui.current.lastAction.PressKey.code === ENTER_KEY_CODE, + ).implies(eventually(() => ui.current.messageText.includes(replyText)).within(30, 'seconds')), + ); +}); + +export const composerEventuallyClearsAfterSubmit = always(() => { + const eligible = ui.current.composerFocused && ui.current.composerValue.trim() !== ''; + return next( + now( + () => + eligible && + typeof ui.current.lastAction === 'object' && + ui.current.lastAction !== null && + 'PressKey' in ui.current.lastAction && + ui.current.lastAction.PressKey.code === ENTER_KEY_CODE, + ).implies(eventually(() => ui.current.composerValue === '').within(5, 'seconds')), + ); +}); + +export const multiConversationSubmissionEventuallyRendersInParallel = always(() => { + const submittedText = ui.current.composerValue.trim(); + const marker = promptMarker(submittedText); + const eligible = + ui.current.hasAddedConversation && ui.current.composerFocused && submittedText !== ''; + return next( + now( + () => + eligible && + typeof ui.current.lastAction === 'object' && + ui.current.lastAction !== null && + 'PressKey' in ui.current.lastAction && + ui.current.lastAction.PressKey.code === ENTER_KEY_CODE, + ).implies( + eventually(() => { + const counts = ui.current.parallelColumnCounts; + return ( + marker !== '' && + ui.current.messageText.includes(marker) && + counts.length > 0 && + counts[counts.length - 1] === 2 + ); + }).within(30, 'seconds'), + ), + ); +}); diff --git a/e2e/bombadil/steering-lifecycle.specification.ts b/e2e/bombadil/steering-lifecycle.specification.ts new file mode 100644 index 0000000000..33eef0ac36 --- /dev/null +++ b/e2e/bombadil/steering-lifecycle.specification.ts @@ -0,0 +1,317 @@ +import { always, eventually, extract, now } from '@antithesishq/bombadil'; +import { actions } from '@antithesishq/bombadil/browser'; +import type { Action, Point, State } from '@antithesishq/bombadil/browser'; +import { + noConsoleErrors, + noHttpErrorCodes, + noUncaughtExceptions, + noUnhandledPromiseRejections, +} from '@antithesishq/bombadil/browser/defaults/properties'; + +type Target = { + name: string; + point: Point; +}; + +const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__'; +const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__'; +const ENTER_KEY_CODE = 13; +const ESCAPE_KEY_CODE = 27; +const PROVIDER = 'Mock Provider C'; +const MODEL = 'mock-model-c'; +const MCP_SERVER = 'E2E Memory'; +const SETUP_PROMPT = 'E2E_REPLY:bombadil-steering-setup'; +const SETUP_REPLY = 'E2E reply bombadil-steering-setup'; +const STEER_LABEL = 'bombadil-steering'; +const STEER_PROMPT = `E2E_STEER_TOOL_REPLY:${STEER_LABEL}`; +const STEER_TEXT = `Steer injection ${STEER_LABEL}`; +const FINAL_REPLY = `E2E steer tool reply done ${STEER_LABEL}`; +let reloadIssued = false; + +function visiblePoint(state: State, element: Element | null): Point | null { + if (!element) { + return null; + } + const style = state.window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.pointerEvents === 'none' || + rect.width <= 0 || + rect.height <= 0 + ) { + return null; + } + const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + const hitElement = state.document.elementFromPoint(point.x, point.y); + if ( + point.x < 0 || + point.y < 0 || + point.x > state.window.innerWidth || + point.y > state.window.innerHeight || + !hitElement || + (hitElement !== element && !element.contains(hitElement)) + ) { + return null; + } + return point; +} + +function target( + state: State, + selector: string, + name: string, + text?: string, + containsText = false, +): Target | null { + for (const element of state.document.querySelectorAll(selector)) { + const content = element.textContent?.trim() ?? ''; + if (text != null && (containsText ? !content.includes(text) : content !== text)) { + continue; + } + if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') { + continue; + } + const point = visiblePoint(state, element); + if (point) { + return { name, point }; + } + } + return null; +} + +function inputValue(state: State, selector: string): string { + return ( + state.document.querySelector(selector)?.value ?? '' + ); +} + +function isFocused(state: State, selector: string): boolean { + return state.document.activeElement?.matches(selector) === true; +} + +function clickOrWait(targetValue: Target | null): Action[] { + return targetValue ? [{ Click: targetValue }] : ['Wait']; +} + +const ui = extract((state: State) => { + const messageElements = Array.from(state.document.querySelectorAll('.message-render')); + const messageText = messageElements.map((element) => element.textContent ?? '').join('\n'); + const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]'); + const duringRunSend = state.document.querySelector('[data-testid="during-run-send-button"]'); + const mcpServerItem = Array.from( + state.document.querySelectorAll('[role="menuitemcheckbox"]'), + ).find((element) => (element.textContent ?? '').includes(MCP_SERVER)); + const selectedMcpButton = Array.from( + state.document.querySelectorAll('button[aria-expanded]'), + ).find((element) => (element.textContent ?? '').includes(MCP_SERVER)); + return { + path: state.window.location.pathname, + lastAction: state.lastAction, + lastActionWasEnter: + typeof state.lastAction === 'object' && + state.lastAction !== null && + 'PressKey' in state.lastAction && + state.lastAction.PressKey.code === ENTER_KEY_CODE, + messageCount: messageElements.length, + messageText, + modelLabel: modelTrigger?.textContent?.trim() ?? '', + composerValue: inputValue(state, '#prompt-textarea'), + composerFocused: isFocused(state, '#prompt-textarea'), + emailValue: inputValue(state, '#email'), + emailFocused: isFocused(state, '#email'), + passwordValue: inputValue(state, '#password'), + passwordFocused: isFocused(state, '#password'), + mcpSelected: + mcpServerItem?.getAttribute('aria-checked') === 'true' || selectedMcpButton !== undefined, + mcpMenuOpen: visiblePoint(state, mcpServerItem ?? null) !== null, + inFlightSteerCount: state.document.querySelectorAll('[data-testid="in-flight-steer"]').length, + appliedSteerCount: state.document.querySelectorAll('[data-testid="steer-part"]').length, + queuedMessageCount: state.document.querySelectorAll('[data-testid="queued-message-row"]') + .length, + finalReplyCount: messageElements.filter((element) => + (element.textContent ?? '').includes(FINAL_REPLY), + ).length, + setupComplete: messageText.includes(SETUP_REPLY), + duringRunAction: duringRunSend?.getAttribute('data-during-run-action') ?? '', + isSubmitting: + state.document.querySelector('[data-testid="stop-generation-button"]') !== null || + duringRunSend !== null, + hasComposer: state.document.querySelector('#prompt-textarea') !== null, + loginEmail: target(state, '#email', 'Login email'), + loginPassword: target(state, '#password', 'Login password'), + loginSubmit: target(state, '[data-testid="login-button"]', 'Login'), + composer: target(state, '#prompt-textarea', 'Message input'), + modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'), + provider: target(state, '[role="option"]', PROVIDER, PROVIDER), + model: target(state, '[role="option"]', MODEL, MODEL), + mcpTrigger: target(state, 'button', 'MCP Servers', 'MCP Servers'), + mcpServer: target(state, '[role="menuitemcheckbox"]', `Select ${MCP_SERVER}`, MCP_SERVER, true), + }; +}); + +export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections }; + +export const steeringLifecycleActions = actions((): Action[] => { + const state = ui.current; + + if (state.path === '/login') { + reloadIssued = false; + if (!state.emailFocused && state.emailValue === '') { + return clickOrWait(state.loginEmail); + } + if (state.emailFocused && state.emailValue === '') { + return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }]; + } + if (!state.passwordFocused && state.passwordValue === '') { + return clickOrWait(state.loginPassword); + } + if (state.passwordFocused && state.passwordValue === '') { + return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }]; + } + return clickOrWait(state.loginSubmit); + } + + if (!state.hasComposer) { + return ['Wait']; + } + + if (state.composerValue.endsWith(STEER_TEXT)) { + if (state.duringRunAction !== 'steer') { + return ['Wait']; + } + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + + if ( + state.setupComplete && + state.lastActionWasEnter && + state.inFlightSteerCount === 0 && + !state.composerValue.endsWith(STEER_TEXT) + ) { + return state.composerFocused + ? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }] + : clickOrWait(state.composer); + } + + if (state.isSubmitting) { + if (!state.setupComplete) { + return ['Wait']; + } + if (state.inFlightSteerCount === 1 || state.appliedSteerCount === 1) { + return ['Wait']; + } + if (state.composerValue === '') { + return state.composerFocused + ? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }] + : clickOrWait(state.composer); + } + if (state.composerValue === STEER_PROMPT) { + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + if (state.duringRunAction !== 'steer') { + return ['Wait']; + } + return state.composerFocused + ? [{ PressKey: { code: ENTER_KEY_CODE } }] + : clickOrWait(state.composer); + } + + if (state.finalReplyCount === 1 && state.appliedSteerCount === 1) { + if (!reloadIssued) { + reloadIssued = true; + return ['Reload']; + } + return ['Wait']; + } + + const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new'; + if (isPersistedConversation && state.messageCount === 0) { + return ['Wait']; + } + + if (state.messageCount === 0) { + if (state.modelLabel === '' || state.modelLabel === 'Select a model') { + return clickOrWait(state.modelTrigger); + } + if (state.model) { + return clickOrWait(state.model); + } + if (state.provider) { + return clickOrWait(state.provider); + } + if (!state.modelLabel.includes(MODEL) && !state.modelLabel.includes(PROVIDER)) { + return clickOrWait(state.modelTrigger); + } + if (!state.mcpSelected) { + return state.mcpServer ? clickOrWait(state.mcpServer) : clickOrWait(state.mcpTrigger); + } + if (state.mcpMenuOpen) { + return [{ PressKey: { code: ESCAPE_KEY_CODE } }]; + } + if (state.composerValue === '') { + return state.composerFocused + ? [{ TypeText: { text: SETUP_PROMPT, delayMillis: 0 } }] + : clickOrWait(state.composer); + } + return state.composerFocused + ? [{ PressKey: { code: ENTER_KEY_CODE } }] + : clickOrWait(state.composer); + } + + if (state.messageCount === 2 && state.setupComplete) { + if (state.composerValue === '') { + return state.composerFocused + ? [{ TypeText: { text: STEER_PROMPT, delayMillis: 0 } }] + : clickOrWait(state.composer); + } + if (!state.composerFocused) { + return clickOrWait(state.composer); + } + return [{ PressKey: { code: ENTER_KEY_CODE } }]; + } + + return ['Wait']; +}); + +/** A submitted steer must be represented immediately while the run is active. */ +export const steerEventuallyBecomesInFlight = eventually( + () => ui.current.inFlightSteerCount === 1, +).within(30, 'seconds'); + +/** The steer drains at the MCP tool boundary and becomes durable in-thread state. */ +export const steerEventuallyApplies = eventually( + () => + ui.current.appliedSteerCount === 1 && + ui.current.inFlightSteerCount === 0 && + ui.current.finalReplyCount === 1, +).within(65, 'seconds'); + +/** A steer is neither duplicated nor degraded into a queued follow-up turn. */ +export const steerStaysSingularAndInBand = always( + () => + ui.current.inFlightSteerCount <= 1 && + ui.current.appliedSteerCount <= 1 && + ui.current.finalReplyCount <= 1 && + ui.current.queuedMessageCount === 0 && + ui.current.messageCount <= 4, +); + +/** The applied steer and terminal response must survive a conversation reload. */ +export const appliedSteerSurvivesReload = always(() => + now(() => ui.current.lastAction === 'Reload').implies( + eventually( + () => + ui.current.appliedSteerCount === 1 && + ui.current.inFlightSteerCount === 0 && + ui.current.finalReplyCount === 1, + ).within(20, 'seconds'), + ), +); diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index cf51932fb9..beae52cfb1 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -211,3 +211,13 @@ modelSpecs: preset: endpoint: 'Mock Provider A' model: 'mock-model-a' + + # Focused Bombadil HITL fixture. The fake model calls ask_user_question + # only for the explicit E2E marker, so enabling the tool here cannot make + # unrelated mock conversations pause. + - name: 'e2e-hitl' + label: 'E2E HITL' + askUserQuestion: true + preset: + endpoint: 'Mock Provider A' + model: 'mock-model-a' diff --git a/e2e/playwright.config.bombadil.ts b/e2e/playwright.config.bombadil.ts new file mode 100644 index 0000000000..dc9a0ca376 --- /dev/null +++ b/e2e/playwright.config.bombadil.ts @@ -0,0 +1,9 @@ +import { defineConfig } from '@playwright/test'; +import mockConfig from './playwright.config.mock'; + +export default defineConfig(mockConfig, { + testDir: 'bombadil', + testMatch: 'harness.spec.ts', + retries: 0, + reporter: [['list']], +}); diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index 801d1ed20a..12debcfffa 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -34,6 +34,7 @@ const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:'; const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:'; const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:'; const ACTIVITY_REPLY_MARKER = 'E2E_ACTIVITY_REPLY:'; +const ASK_USER_QUESTION_MARKER = 'E2E_ASK_USER_QUESTION:'; const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:'; const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:'; const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY'; @@ -68,6 +69,7 @@ const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done'; const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued'; const ACTIVITY_FINAL_TEXT = 'E2E activity reply done'; const STEER_TOOL_NAME_PREFIX = 'remember_fact'; +const ASK_USER_QUESTION_TOOL_NAME = 'ask_user_question'; const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35; const ORDERED_CHUNK_DELAY_MS = 2; const ORDERED_REPLY_PIECES = 64; @@ -1148,6 +1150,41 @@ function activityReplyResponses(label, toolNames) { }; } +/** + * Pause a real agent run at the ask_user_question tool. The resume controller + * rebuilds the graph with an empty input-message list, so the test hook selects + * its ordinary mock reply for the resumed model turn. This deliberately tests + * the production checkpoint/resume seam rather than simulating a pause in the + * browser fixture. + */ +function askUserQuestionResponses(label, toolNames) { + if (!toolNames.has(ASK_USER_QUESTION_TOOL_NAME)) { + return { + responses: [ + `E2E ask user question unavailable: ${ASK_USER_QUESTION_TOOL_NAME} was not advertised.`, + ], + }; + } + return { + responses: [''], + toolCalls: [ + { + id: `call_e2e_ask_user_question_${label}`, + name: ASK_USER_QUESTION_TOOL_NAME, + args: { + question: `Which environment should Bombadil use for ${label}?`, + description: 'This deterministic pause exercises the HITL answer and resume lifecycle.', + options: [ + { label: 'Staging', value: 'staging' }, + { label: 'Production', value: 'production' }, + ], + }, + type: 'tool_call', + }, + ], + }; +} + function findLastToolMessageText(messages, requiredToken) { for (let index = (messages ?? []).length - 1; index >= 0; index--) { const message = messages[index]; @@ -2023,6 +2060,11 @@ function resolveResponses({ graph, messages, text, toolNames }) { return activityReplyResponses(activityLabel, toolNames); } + const askUserQuestionLabel = getMarkerValue(text, ASK_USER_QUESTION_MARKER); + if (askUserQuestionLabel) { + return askUserQuestionResponses(askUserQuestionLabel, toolNames); + } + if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) { return { responses: [MOCK_REPLY], diff --git a/package-lock.json b/package-lock.json index a2c52cc939..52f891bc89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "packages/*" ], "devDependencies": { + "@antithesishq/bombadil": "0.6.1", "@axe-core/playwright": "^4.10.1", "@eslint/compat": "^1.2.6", "@eslint/eslintrc": "^3.3.4", @@ -1255,6 +1256,16 @@ "google-auth-library": "^9.4.2" } }, + "node_modules/@antithesishq/bombadil": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz", + "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==", + "dev": true, + "license": "MIT", + "bin": { + "bombadil": "bin/bombadil.js" + } + }, "node_modules/@apideck/better-ajv-errors": { "version": "0.3.7", "resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.7.tgz", diff --git a/package.json b/package.json index 172c6b7a60..32cbb0a109 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,18 @@ "e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", "e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts", "e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts", + "e2e:bombadil": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:run": "playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:branch-reload": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:branch-reload:run": "cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:fork-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:fork-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:model-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:model-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:hitl": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:hitl:run": "cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:steering": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts", + "e2e:bombadil:steering:run": "cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts", "e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts", "e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", "e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts", @@ -134,6 +146,7 @@ }, "homepage": "https://librechat.ai/", "devDependencies": { + "@antithesishq/bombadil": "0.6.1", "@axe-core/playwright": "^4.10.1", "@eslint/compat": "^1.2.6", "@eslint/eslintrc": "^3.3.4",