mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-27 04:07:05 +00:00
🧪 test(e2e): add Bombadil property exploration (#14462)
* test(e2e): add Bombadil property exploration * fix(e2e): address Bombadil review feedback
This commit is contained in:
Ravi Kumar L
GitHub
parent
54d7f04d71
commit
7cf4c3f73f
15 changed files with 2288 additions and 0 deletions
180
.github/workflows/playwright-bombadil.yml
vendored
Normal file
180
.github/workflows/playwright-bombadil.yml
vendored
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
name: Bombadil Property Exploration
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '**'
|
||||
- '!**.md'
|
||||
- '!.github/workflows/**'
|
||||
- '.github/workflows/playwright-bombadil.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
reason:
|
||||
description: 'Reason for manual trigger'
|
||||
required: false
|
||||
default: 'Manual Bombadil run'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-bombadil-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}'
|
||||
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
|
||||
|
||||
jobs:
|
||||
bombadil:
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request != null &&
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
BOMBADIL_TIME_LIMIT: '300s'
|
||||
E2E_CHROMIUM_CHANNEL: chrome
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Use Node.js 24.16.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24.16.0'
|
||||
|
||||
- name: Restore node_modules cache
|
||||
id: cache-node-modules
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
node_modules
|
||||
client/node_modules
|
||||
packages/client/node_modules
|
||||
packages/data-provider/node_modules
|
||||
packages/data-schemas/node_modules
|
||||
packages/api/node_modules
|
||||
api/node_modules
|
||||
key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.cache-node-modules.outputs.cache-hit != 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Restore data-provider build cache
|
||||
id: cache-data-provider
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/data-provider/dist
|
||||
key: build-data-provider-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build data-provider
|
||||
if: steps.cache-data-provider.outputs.cache-hit != 'true'
|
||||
run: npm run build:data-provider
|
||||
|
||||
- name: Restore data-schemas build cache
|
||||
id: cache-data-schemas
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/data-schemas/dist
|
||||
key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build data-schemas
|
||||
if: steps.cache-data-schemas.outputs.cache-hit != 'true'
|
||||
run: npm run build:data-schemas
|
||||
|
||||
- name: Restore api build cache
|
||||
id: cache-api
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/api/dist
|
||||
key: build-api-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/tsdown.config.mjs', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json') }}
|
||||
|
||||
- name: Build api
|
||||
if: steps.cache-api.outputs.cache-hit != 'true'
|
||||
run: npm run build:api
|
||||
|
||||
- name: Restore client-package build cache
|
||||
id: cache-client-package
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packages/client/dist
|
||||
key: build-client-package-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build client-package
|
||||
if: steps.cache-client-package.outputs.cache-hit != 'true'
|
||||
run: npm run build:client-package
|
||||
|
||||
- name: Restore client app build cache
|
||||
id: cache-client-app
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: client/dist
|
||||
key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package-lock.json', 'client/src/**', 'client/public/**', 'client/scripts/post-build.cjs', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
|
||||
|
||||
- name: Build client app
|
||||
if: steps.cache-client-app.outputs.cache-hit != 'true'
|
||||
run: npm run build:client
|
||||
|
||||
- name: Install Playwright runtime dependencies
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
google-chrome --version
|
||||
npx playwright install-deps chrome
|
||||
|
||||
- name: Run five-minute Bombadil exploration
|
||||
id: bombadil
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p e2e/.generated
|
||||
npx playwright test \
|
||||
--config=e2e/playwright.config.bombadil.ts \
|
||||
--reporter=line,html \
|
||||
2>&1 | tee e2e/.generated/bombadil-ci.log
|
||||
env:
|
||||
CI: 'true'
|
||||
PLAYWRIGHT_HTML_OPEN: 'never'
|
||||
PLAYWRIGHT_HTML_OUTPUT_DIR: e2e/playwright-report-bombadil
|
||||
|
||||
- name: Upload Bombadil reproduction trace
|
||||
id: bombadil-reproduction
|
||||
if: steps.bombadil.outcome == 'failure'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bombadil-reproduction-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: e2e/.generated/bombadil-output/**
|
||||
include-hidden-files: true
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload Bombadil diagnostics
|
||||
id: bombadil-diagnostics
|
||||
if: steps.bombadil.outcome == 'failure'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bombadil-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
e2e/.generated/bombadil-ci.log
|
||||
e2e/playwright-report-bombadil/**
|
||||
e2e/specs/.test-results/**
|
||||
include-hidden-files: true
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Report non-blocking Bombadil failure
|
||||
if: steps.bombadil.outcome == 'failure'
|
||||
run: |
|
||||
echo "::warning title=Bombadil property violation::The five-minute exploration failed. Download the reproduction and diagnostics artifacts for this run."
|
||||
{
|
||||
echo "### Bombadil property exploration"
|
||||
echo
|
||||
echo "The exploration failed, but this job does not block merge."
|
||||
echo
|
||||
echo "Reproduction: ${{ steps.bombadil-reproduction.outputs.artifact-url }}"
|
||||
echo
|
||||
echo "Diagnostics: ${{ steps.bombadil-diagnostics.outputs.artifact-url }}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -96,6 +96,7 @@ src/style - official.css
|
|||
/e2e/benchmarks/.test-results/
|
||||
/e2e/.generated/
|
||||
/e2e/playwright-report/
|
||||
/e2e/playwright-report-bombadil/
|
||||
/playwright/.cache/
|
||||
.DS_Store
|
||||
*.code-workspace
|
||||
|
|
|
|||
|
|
@ -18,6 +18,76 @@ CI runs the complete mock suite in both stream modes. Each mode is split across
|
|||
npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4
|
||||
```
|
||||
|
||||
## Property-based browser testing
|
||||
|
||||
Bombadil explores randomized sequences across the core chat loop, message branches,
|
||||
parallel multi-conversation responses, model changes, reloads, and sidebar conversation
|
||||
lifecycle operations:
|
||||
|
||||
```sh
|
||||
npm run e2e:bombadil
|
||||
```
|
||||
|
||||
Set `BOMBADIL_TIME_LIMIT` for longer local or scheduled runs. Failures leave a
|
||||
reproducible trace under `e2e/.generated/bombadil-output`; rerun it with:
|
||||
|
||||
```sh
|
||||
BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run
|
||||
```
|
||||
|
||||
Reproducing a real violation is expected to fail the Playwright test. Before a
|
||||
new run overwrites the active output, the harness archives it under
|
||||
`e2e/.generated/bombadil-history/`. Reproduction can diverge when streaming
|
||||
timing changes; Bombadil reports that explicitly.
|
||||
|
||||
The harness uses the credential-free mock-LLM profile, so exploration never sends
|
||||
billable provider requests.
|
||||
|
||||
CI runs the broad property exploration for five minutes in the non-blocking
|
||||
`Bombadil Property Exploration` workflow. If a property fails, download the
|
||||
`bombadil-reproduction-*` artifact into
|
||||
`e2e/.generated/bombadil-output/`, then reproduce it locally:
|
||||
|
||||
```sh
|
||||
BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run
|
||||
```
|
||||
|
||||
The accompanying `bombadil-diagnostics-*` artifact contains the captured CI log,
|
||||
Playwright HTML report, and Playwright test results. A Bombadil failure produces
|
||||
a workflow warning but does not block merge.
|
||||
|
||||
The default instruments inline JavaScript only because instrumenting LibreChat's
|
||||
full Vite bundle can exceed Bombadil's driver timeout during stateful runs. Set
|
||||
`BOMBADIL_INSTRUMENT_JAVASCRIPT=files,inline` for shorter coverage-guided
|
||||
experiments.
|
||||
|
||||
The branch reload, fork submission, model/conversation, HITL pause/resume, and
|
||||
mid-run steering lifecycle properties can be run independently:
|
||||
|
||||
```sh
|
||||
npm run e2e:bombadil:branch-reload
|
||||
npm run e2e:bombadil:fork-lifecycle
|
||||
npm run e2e:bombadil:model-lifecycle
|
||||
npm run e2e:bombadil:hitl
|
||||
npm run e2e:bombadil:steering
|
||||
```
|
||||
|
||||
These focused commands are diagnostic properties: they exit nonzero when they
|
||||
reproduce a product invariant violation. Reproduce a focused trace with its
|
||||
matching `:run` script and output directory, for example:
|
||||
|
||||
```sh
|
||||
BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output-hitl npm run e2e:bombadil:hitl:run
|
||||
```
|
||||
|
||||
HITL drives a real `ask_user_question` checkpoint through the answer/resume
|
||||
controller, reloads while the question is paused, answers it once, and reloads
|
||||
the completed conversation. Steering submits an in-flight steer during a slow
|
||||
MCP-backed run, checks that it moves exactly once from the composer anchor into
|
||||
the response at the tool boundary, and reloads the applied state. The model
|
||||
lifecycle property is the passing control. The branch reload and fork
|
||||
properties preserve their minimal failing traces.
|
||||
|
||||
## Recording Tests
|
||||
|
||||
Use Playwright codegen when you want to turn an exploratory browser session into a draft test:
|
||||
|
|
|
|||
228
e2e/bombadil/branch-reload.specification.ts
Normal file
228
e2e/bombadil/branch-reload.specification.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
import { always, eventually, extract } from '@antithesishq/bombadil';
|
||||
import { actions } from '@antithesishq/bombadil/browser';
|
||||
import type { Action, Point, State } from '@antithesishq/bombadil/browser';
|
||||
import {
|
||||
noConsoleErrors,
|
||||
noHttpErrorCodes,
|
||||
noUncaughtExceptions,
|
||||
noUnhandledPromiseRejections,
|
||||
} from '@antithesishq/bombadil/browser/defaults/properties';
|
||||
|
||||
type Target = {
|
||||
name: string;
|
||||
point: Point;
|
||||
};
|
||||
|
||||
type NavigationStatus = {
|
||||
current: number;
|
||||
total: number;
|
||||
};
|
||||
|
||||
const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
const ENTER_KEY_CODE = 13;
|
||||
const BRANCH_PROMPT = 'E2E_REPLY:bombadil-branch-reload';
|
||||
const RELOAD_MARKER = 'bombadil-reload-marker';
|
||||
|
||||
function visiblePoint(state: State, element: Element | null): Point | null {
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
const style = state.window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.pointerEvents === 'none' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const point = {
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
};
|
||||
if (
|
||||
point.x < 0 ||
|
||||
point.y < 0 ||
|
||||
point.x > state.window.innerWidth ||
|
||||
point.y > state.window.innerHeight
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const hitElement = state.document.elementFromPoint(point.x, point.y);
|
||||
if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
|
||||
return null;
|
||||
}
|
||||
return point;
|
||||
}
|
||||
|
||||
function target(state: State, selector: string, name: string, last = false): Target | null {
|
||||
const candidates = Array.from(state.document.querySelectorAll(selector));
|
||||
const elements = last ? candidates.reverse() : candidates;
|
||||
for (const element of elements) {
|
||||
if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
|
||||
continue;
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
if (point) {
|
||||
return { name, point };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function inputValue(state: State, selector: string): string {
|
||||
const element = state.document.querySelector(selector) as
|
||||
| HTMLInputElement
|
||||
| HTMLTextAreaElement
|
||||
| null;
|
||||
return element?.value ?? '';
|
||||
}
|
||||
|
||||
function isFocused(state: State, selector: string): boolean {
|
||||
return state.document.activeElement?.matches(selector) === true;
|
||||
}
|
||||
|
||||
function clickOrWait(targetValue: Target | null): Action[] {
|
||||
return targetValue ? [{ Click: targetValue }] : ['Wait'];
|
||||
}
|
||||
|
||||
function isNamedClick(action: Action | null, name: string): boolean {
|
||||
return (
|
||||
typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name
|
||||
);
|
||||
}
|
||||
|
||||
const ui = extract((state: State) => {
|
||||
const statuses: NavigationStatus[] = [];
|
||||
for (const navigation of state.document.querySelectorAll(
|
||||
'nav[aria-label="Sibling message navigation"]',
|
||||
)) {
|
||||
const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? '';
|
||||
const match = text.match(/^(\d+)\s*\/\s*(\d+)$/);
|
||||
if (match) {
|
||||
statuses.push({ current: Number(match[1]), total: Number(match[2]) });
|
||||
}
|
||||
}
|
||||
const encodedReloadMarker = state.window.btoa(RELOAD_MARKER);
|
||||
const hasSavedReloadMarker = Object.keys(state.window.localStorage).some(
|
||||
(key) =>
|
||||
key.startsWith('textDraft_') &&
|
||||
state.window.localStorage.getItem(key) === encodedReloadMarker,
|
||||
);
|
||||
|
||||
return {
|
||||
path: state.window.location.pathname,
|
||||
lastAction: state.lastAction,
|
||||
messageCount: state.document.querySelectorAll('.message-render').length,
|
||||
statuses,
|
||||
hasSavedReloadMarker,
|
||||
composerValue: inputValue(state, '#prompt-textarea'),
|
||||
composerFocused: isFocused(state, '#prompt-textarea'),
|
||||
emailValue: inputValue(state, '#email'),
|
||||
emailFocused: isFocused(state, '#email'),
|
||||
passwordValue: inputValue(state, '#password'),
|
||||
passwordFocused: isFocused(state, '#password'),
|
||||
isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
|
||||
hasComposer: state.document.querySelector('#prompt-textarea') !== null,
|
||||
loginEmail: target(state, '#email', 'Login email'),
|
||||
loginPassword: target(state, '#password', 'Login password'),
|
||||
loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
|
||||
composer: target(state, '#prompt-textarea', 'Message input'),
|
||||
regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', true),
|
||||
previousSibling: target(
|
||||
state,
|
||||
'button[aria-label="Previous sibling message"]',
|
||||
'Previous sibling message',
|
||||
true,
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
|
||||
|
||||
export const branchReloadActions = actions((): Action[] => {
|
||||
const state = ui.current;
|
||||
|
||||
if (state.path === '/login') {
|
||||
if (!state.emailFocused && state.emailValue === '') {
|
||||
return clickOrWait(state.loginEmail);
|
||||
}
|
||||
if (state.emailFocused && state.emailValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.passwordFocused && state.passwordValue === '') {
|
||||
return clickOrWait(state.loginPassword);
|
||||
}
|
||||
if (state.passwordFocused && state.passwordValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.loginSubmit);
|
||||
}
|
||||
|
||||
if (state.isSubmitting || !state.hasComposer) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
|
||||
if (isPersistedConversation && state.messageCount === 0) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.messageCount === 0) {
|
||||
if (state.composerValue === '') {
|
||||
if (state.composerFocused) {
|
||||
return [{ TypeText: { text: BRANCH_PROMPT, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
|
||||
const status = state.statuses[state.statuses.length - 1];
|
||||
if (state.lastAction === 'Reload') {
|
||||
return ['Wait'];
|
||||
}
|
||||
if (!status) {
|
||||
if (isNamedClick(state.lastAction, 'Regenerate')) {
|
||||
return ['Wait'];
|
||||
}
|
||||
return clickOrWait(state.regenerate);
|
||||
}
|
||||
if (status.total === 2 && status.current === 2) {
|
||||
if (state.hasSavedReloadMarker) {
|
||||
return ['Wait'];
|
||||
}
|
||||
return clickOrWait(state.previousSibling);
|
||||
}
|
||||
if (status.total === 2 && status.current === 1) {
|
||||
if (state.composerValue !== RELOAD_MARKER) {
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ TypeText: { text: RELOAD_MARKER, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.hasSavedReloadMarker) {
|
||||
return ['Wait'];
|
||||
}
|
||||
return ['Reload'];
|
||||
}
|
||||
return ['Wait'];
|
||||
});
|
||||
|
||||
export const siblingBranchEventuallyExists = eventually(() =>
|
||||
ui.current.statuses.some(({ total }) => total === 2),
|
||||
).within(45, 'seconds');
|
||||
|
||||
export const selectedSiblingSurvivesReload = always(
|
||||
() =>
|
||||
!ui.current.hasSavedReloadMarker ||
|
||||
ui.current.messageCount === 0 ||
|
||||
!ui.current.statuses.some(({ total }) => total === 2) ||
|
||||
ui.current.statuses.some(({ current, total }) => current === 1 && total === 2),
|
||||
);
|
||||
202
e2e/bombadil/fork-lifecycle.specification.ts
Normal file
202
e2e/bombadil/fork-lifecycle.specification.ts
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
import { always, eventually, extract, next, now } from '@antithesishq/bombadil';
|
||||
import { actions } from '@antithesishq/bombadil/browser';
|
||||
import type { Action, Point, State } from '@antithesishq/bombadil/browser';
|
||||
import {
|
||||
noConsoleErrors,
|
||||
noHttpErrorCodes,
|
||||
noUncaughtExceptions,
|
||||
noUnhandledPromiseRejections,
|
||||
} from '@antithesishq/bombadil/browser/defaults/properties';
|
||||
|
||||
type Target = {
|
||||
name: string;
|
||||
point: Point;
|
||||
};
|
||||
|
||||
const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
const ENTER_KEY_CODE = 13;
|
||||
const FORK_PROMPT = 'E2E_REPLY:bombadil-fork-lifecycle';
|
||||
|
||||
function visiblePoint(state: State, element: Element | null): Point | null {
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
const style = state.window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.pointerEvents === 'none' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const point = {
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
};
|
||||
if (
|
||||
point.x < 0 ||
|
||||
point.y < 0 ||
|
||||
point.x > state.window.innerWidth ||
|
||||
point.y > state.window.innerHeight
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const hitElement = state.document.elementFromPoint(point.x, point.y);
|
||||
if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
|
||||
return null;
|
||||
}
|
||||
return point;
|
||||
}
|
||||
|
||||
function target(
|
||||
state: State,
|
||||
selector: string,
|
||||
name: string,
|
||||
text?: string,
|
||||
last = false,
|
||||
): Target | null {
|
||||
const candidates = Array.from(state.document.querySelectorAll(selector)).filter(
|
||||
(element) => text == null || element.textContent?.trim() === text,
|
||||
);
|
||||
const elements = last ? candidates.reverse() : candidates;
|
||||
for (const element of elements) {
|
||||
if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
|
||||
continue;
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
if (point) {
|
||||
return { name, point };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function inputValue(state: State, selector: string): string {
|
||||
const element = state.document.querySelector(selector) as
|
||||
| HTMLInputElement
|
||||
| HTMLTextAreaElement
|
||||
| null;
|
||||
return element?.value ?? '';
|
||||
}
|
||||
|
||||
function isFocused(state: State, selector: string): boolean {
|
||||
return state.document.activeElement?.matches(selector) === true;
|
||||
}
|
||||
|
||||
function clickOrWait(targetValue: Target | null): Action[] {
|
||||
return targetValue ? [{ Click: targetValue }] : ['Wait'];
|
||||
}
|
||||
|
||||
function isNamedClick(action: Action | null, name: string): boolean {
|
||||
return (
|
||||
typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name
|
||||
);
|
||||
}
|
||||
|
||||
const ui = extract((state: State) => ({
|
||||
path: state.window.location.pathname,
|
||||
lastAction: state.lastAction,
|
||||
bodyText: state.document.body.textContent ?? '',
|
||||
messageCount: state.document.querySelectorAll('.message-render').length,
|
||||
composerValue: inputValue(state, '#prompt-textarea'),
|
||||
composerFocused: isFocused(state, '#prompt-textarea'),
|
||||
emailValue: inputValue(state, '#email'),
|
||||
emailFocused: isFocused(state, '#email'),
|
||||
passwordValue: inputValue(state, '#password'),
|
||||
passwordFocused: isFocused(state, '#password'),
|
||||
isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
|
||||
hasComposer: state.document.querySelector('#prompt-textarea') !== null,
|
||||
loginEmail: target(state, '#email', 'Login email'),
|
||||
loginPassword: target(state, '#password', 'Login password'),
|
||||
loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
|
||||
composer: target(state, '#prompt-textarea', 'Message input'),
|
||||
forkMenu: target(state, 'button[aria-label="Open Fork Menu"]', 'Open fork menu', undefined, true),
|
||||
forkVisible: target(state, 'button', 'Fork visible messages', 'Visible messages only', true),
|
||||
}));
|
||||
|
||||
export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
|
||||
|
||||
export const forkLifecycleActions = actions((): Action[] => {
|
||||
const state = ui.current;
|
||||
|
||||
if (state.path === '/login') {
|
||||
if (!state.emailFocused && state.emailValue === '') {
|
||||
return clickOrWait(state.loginEmail);
|
||||
}
|
||||
if (state.emailFocused && state.emailValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.passwordFocused && state.passwordValue === '') {
|
||||
return clickOrWait(state.loginPassword);
|
||||
}
|
||||
if (state.passwordFocused && state.passwordValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.loginSubmit);
|
||||
}
|
||||
|
||||
if (state.isSubmitting || !state.hasComposer) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
|
||||
if (isPersistedConversation && state.messageCount === 0) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.messageCount === 0) {
|
||||
if (state.composerValue === '') {
|
||||
if (state.composerFocused) {
|
||||
return [{ TypeText: { text: FORK_PROMPT, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
|
||||
if (state.forkVisible) {
|
||||
return clickOrWait(state.forkVisible);
|
||||
}
|
||||
return clickOrWait(state.forkMenu);
|
||||
});
|
||||
|
||||
export const forkSetupEventuallyReachesChoice = eventually(
|
||||
() => ui.current.forkVisible !== null,
|
||||
).within(45, 'seconds');
|
||||
|
||||
export const forkChoiceBecomesUnavailableAfterSubmission = always(() => {
|
||||
const choiceWasVisible = ui.current.forkVisible !== null;
|
||||
return next(
|
||||
now(
|
||||
() => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'),
|
||||
).implies(now(() => ui.current.forkVisible === null)),
|
||||
);
|
||||
});
|
||||
|
||||
export const forkSubmissionEventuallyNavigates = always(() => {
|
||||
const originalPath = ui.current.path;
|
||||
const choiceWasVisible = ui.current.forkVisible !== null;
|
||||
return next(
|
||||
now(
|
||||
() => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'),
|
||||
).implies(
|
||||
eventually(
|
||||
() =>
|
||||
ui.current.path.startsWith('/c/') &&
|
||||
ui.current.path !== '/c/new' &&
|
||||
ui.current.path !== originalPath,
|
||||
).within(30, 'seconds'),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
export const forkRateLimitIsNeverReached = always(
|
||||
() => !ui.current.bodyText.includes('Too many fork requests. Please try again later'),
|
||||
);
|
||||
176
e2e/bombadil/harness.spec.ts
Normal file
176
e2e/bombadil/harness.spec.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import { spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { getE2EBaseURL } from '../setup/env';
|
||||
import { getE2EUser } from '../setup/user';
|
||||
|
||||
const rootPath = path.resolve(__dirname, '../..');
|
||||
const specificationPath = path.resolve(
|
||||
__dirname,
|
||||
process.env.BOMBADIL_SPECIFICATION ?? 'specification.ts',
|
||||
);
|
||||
const specificationStem = path.basename(specificationPath, '.ts').replace(/\.specification$/, '');
|
||||
const defaultOutputPath = path.resolve(
|
||||
rootPath,
|
||||
specificationStem === 'specification'
|
||||
? 'e2e/.generated/bombadil-output'
|
||||
: `e2e/.generated/bombadil-output-${specificationStem}`,
|
||||
);
|
||||
const defaultBinaryPath = path.resolve(rootPath, 'node_modules/.bin/bombadil');
|
||||
const MAX_CAPTURED_OUTPUT = 100_000;
|
||||
const DEFAULT_REPRODUCTION_TIMEOUT_MS = 30 * 60 * 1_000;
|
||||
const LOGIN_EMAIL_PLACEHOLDER = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD_PLACEHOLDER = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
|
||||
function durationMillis(value: string): number | null {
|
||||
const match = value.match(/^(\d+)(s|m|h|d)$/);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const factors = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 };
|
||||
return Number(match[1]) * factors[match[2] as keyof typeof factors];
|
||||
}
|
||||
|
||||
function positiveTimeoutMillis(value: string, variableName: string): number {
|
||||
const timeout = Number(value);
|
||||
if (!Number.isFinite(timeout) || timeout <= 0) {
|
||||
throw new Error(
|
||||
`${variableName} must be a positive number of milliseconds; received "${value}".`,
|
||||
);
|
||||
}
|
||||
return timeout;
|
||||
}
|
||||
|
||||
function archiveExistingOutput(outputPath: string): void {
|
||||
if (!fs.existsSync(outputPath)) {
|
||||
return;
|
||||
}
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const archivePath = path.resolve(
|
||||
rootPath,
|
||||
'e2e/.generated/bombadil-history',
|
||||
`${path.basename(outputPath)}-${timestamp}`,
|
||||
);
|
||||
fs.mkdirSync(path.dirname(archivePath), { recursive: true });
|
||||
fs.cpSync(outputPath, archivePath, { recursive: true });
|
||||
}
|
||||
|
||||
function materializeSpecification(): string {
|
||||
const user = getE2EUser();
|
||||
const generatedPath = path.resolve(
|
||||
rootPath,
|
||||
'e2e/.generated/bombadil-specifications',
|
||||
path.basename(specificationPath),
|
||||
);
|
||||
const replaceStringLiteral = (source: string, placeholder: string, value: string) =>
|
||||
source
|
||||
.replaceAll(`'${placeholder}'`, JSON.stringify(value))
|
||||
.replaceAll(`"${placeholder}"`, JSON.stringify(value));
|
||||
let source = fs.readFileSync(specificationPath, 'utf8');
|
||||
source = replaceStringLiteral(source, LOGIN_EMAIL_PLACEHOLDER, user.email);
|
||||
source = replaceStringLiteral(source, LOGIN_PASSWORD_PLACEHOLDER, user.password);
|
||||
fs.mkdirSync(path.dirname(generatedPath), { recursive: true });
|
||||
fs.writeFileSync(generatedPath, source);
|
||||
return generatedPath;
|
||||
}
|
||||
|
||||
function runBombadil(
|
||||
outputPath: string,
|
||||
runtimeSpecificationPath: string,
|
||||
childTimeoutMillis: number,
|
||||
): Promise<{ code: number | null; output: string }> {
|
||||
const binaryPath = process.env.BOMBADIL_BIN ?? defaultBinaryPath;
|
||||
const reproducePath = process.env.BOMBADIL_REPRODUCE;
|
||||
const timeLimit = process.env.BOMBADIL_TIME_LIMIT ?? '90s';
|
||||
const args = [
|
||||
'browser',
|
||||
'test',
|
||||
'--headless',
|
||||
'--output-path',
|
||||
outputPath,
|
||||
'--output-path-overwrite',
|
||||
'--instrument-javascript',
|
||||
// Instrumenting LibreChat's full Vite bundle makes Bombadil's driver time out
|
||||
// under longer stateful runs. Full `files,inline` coverage remains opt-in.
|
||||
process.env.BOMBADIL_INSTRUMENT_JAVASCRIPT ?? 'inline',
|
||||
];
|
||||
|
||||
if (reproducePath) {
|
||||
args.push('--reproduce', reproducePath);
|
||||
} else {
|
||||
args.push('--exit-on-violation', '--time-limit', timeLimit);
|
||||
}
|
||||
|
||||
args.push(getE2EBaseURL(), runtimeSpecificationPath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(binaryPath, args, {
|
||||
cwd: rootPath,
|
||||
env: {
|
||||
...process.env,
|
||||
RUST_LOG: process.env.BOMBADIL_RUST_LOG ?? 'error',
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let output = '';
|
||||
let timedOut = false;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
const childTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
output = `${output}\nBombadil exceeded the ${childTimeoutMillis}ms child-process timeout.`;
|
||||
child.kill('SIGTERM');
|
||||
forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 5_000);
|
||||
}, childTimeoutMillis);
|
||||
const append = (chunk: Buffer) => {
|
||||
const text = chunk.toString();
|
||||
output = `${output}${text}`.slice(-MAX_CAPTURED_OUTPUT);
|
||||
process.stdout.write(text);
|
||||
};
|
||||
|
||||
child.stdout.on('data', append);
|
||||
child.stderr.on('data', append);
|
||||
child.once('error', (error) => {
|
||||
clearTimeout(childTimer);
|
||||
if (forceKillTimer) {
|
||||
clearTimeout(forceKillTimer);
|
||||
}
|
||||
reject(error);
|
||||
});
|
||||
child.once('exit', (code) => {
|
||||
clearTimeout(childTimer);
|
||||
if (forceKillTimer) {
|
||||
clearTimeout(forceKillTimer);
|
||||
}
|
||||
resolve({ code: timedOut ? null : code, output });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test('Bombadil explores core, branching, multi-conversation, and lifecycle flows', async () => {
|
||||
const timeLimitMillis = durationMillis(process.env.BOMBADIL_TIME_LIMIT ?? '90s') ?? 90_000;
|
||||
const defaultHarnessTimeout = process.env.BOMBADIL_REPRODUCE
|
||||
? DEFAULT_REPRODUCTION_TIMEOUT_MS
|
||||
: timeLimitMillis + 120_000;
|
||||
const harnessTimeout = process.env.BOMBADIL_HARNESS_TIMEOUT_MS
|
||||
? positiveTimeoutMillis(process.env.BOMBADIL_HARNESS_TIMEOUT_MS, 'BOMBADIL_HARNESS_TIMEOUT_MS')
|
||||
: defaultHarnessTimeout;
|
||||
test.setTimeout(harnessTimeout);
|
||||
const outputPath = process.env.BOMBADIL_REPRODUCE
|
||||
? path.resolve(rootPath, 'e2e/.generated/bombadil-reproduction')
|
||||
: defaultOutputPath;
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
archiveExistingOutput(outputPath);
|
||||
const runtimeSpecificationPath = materializeSpecification();
|
||||
|
||||
const result = await runBombadil(
|
||||
outputPath,
|
||||
runtimeSpecificationPath,
|
||||
Math.max(harnessTimeout - 10_000, 10_000),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.code,
|
||||
`Bombadil exited with ${result.code}. Inspect or reproduce the trace at ${outputPath}.\n${result.output}`,
|
||||
).toBe(0);
|
||||
});
|
||||
248
e2e/bombadil/hitl-lifecycle.specification.ts
Normal file
248
e2e/bombadil/hitl-lifecycle.specification.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
import { always, eventually, extract, now } from '@antithesishq/bombadil';
|
||||
import { actions } from '@antithesishq/bombadil/browser';
|
||||
import type { Action, Point, State } from '@antithesishq/bombadil/browser';
|
||||
import {
|
||||
noConsoleErrors,
|
||||
noHttpErrorCodes,
|
||||
noUncaughtExceptions,
|
||||
noUnhandledPromiseRejections,
|
||||
} from '@antithesishq/bombadil/browser/defaults/properties';
|
||||
|
||||
type Target = {
|
||||
name: string;
|
||||
point: Point;
|
||||
};
|
||||
|
||||
const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
const ENTER_KEY_CODE = 13;
|
||||
const HITL_MODEL_SPEC = 'E2E HITL';
|
||||
const HITL_LABEL = 'bombadil-hitl';
|
||||
const HITL_PROMPT = `E2E_ASK_USER_QUESTION:${HITL_LABEL}`;
|
||||
const HITL_QUESTION = `Which environment should Bombadil use for ${HITL_LABEL}?`;
|
||||
const HITL_OPTION = 'Staging';
|
||||
const FINAL_REPLY = 'E2E mock reply: pong';
|
||||
const COMPLETED_ANSWER = 'You answered: Staging';
|
||||
let reloadIssued = false;
|
||||
let pausedReloadIssued = false;
|
||||
|
||||
function visiblePoint(state: State, element: Element | null): Point | null {
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
const style = state.window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.pointerEvents === 'none' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
|
||||
const hitElement = state.document.elementFromPoint(point.x, point.y);
|
||||
if (
|
||||
point.x < 0 ||
|
||||
point.y < 0 ||
|
||||
point.x > state.window.innerWidth ||
|
||||
point.y > state.window.innerHeight ||
|
||||
!hitElement ||
|
||||
(hitElement !== element && !element.contains(hitElement))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return point;
|
||||
}
|
||||
|
||||
function target(
|
||||
state: State,
|
||||
selector: string,
|
||||
name: string,
|
||||
text?: string,
|
||||
containsText = false,
|
||||
): Target | null {
|
||||
for (const element of state.document.querySelectorAll(selector)) {
|
||||
const content = element.textContent?.trim() ?? '';
|
||||
if (text != null && (containsText ? !content.includes(text) : content !== text)) {
|
||||
continue;
|
||||
}
|
||||
if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
|
||||
continue;
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
if (point) {
|
||||
return { name, point };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function visibleTextCount(
|
||||
state: State,
|
||||
selector: string,
|
||||
text: string,
|
||||
containsText = false,
|
||||
): number {
|
||||
return Array.from(state.document.querySelectorAll(selector)).filter((element) => {
|
||||
const content = element.textContent?.trim() ?? '';
|
||||
return (
|
||||
(containsText ? content.includes(text) : content === text) &&
|
||||
visiblePoint(state, element) !== null
|
||||
);
|
||||
}).length;
|
||||
}
|
||||
|
||||
function inputValue(state: State, selector: string): string {
|
||||
return (
|
||||
state.document.querySelector<HTMLInputElement | HTMLTextAreaElement>(selector)?.value ?? ''
|
||||
);
|
||||
}
|
||||
|
||||
function isFocused(state: State, selector: string): boolean {
|
||||
return state.document.activeElement?.matches(selector) === true;
|
||||
}
|
||||
|
||||
function clickOrWait(targetValue: Target | null): Action[] {
|
||||
return targetValue ? [{ Click: targetValue }] : ['Wait'];
|
||||
}
|
||||
|
||||
const ui = extract((state: State) => {
|
||||
const messageElements = Array.from(state.document.querySelectorAll('.message-render'));
|
||||
const messageText = messageElements.map((element) => element.textContent ?? '').join('\n');
|
||||
const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
|
||||
return {
|
||||
path: state.window.location.pathname,
|
||||
lastAction: state.lastAction,
|
||||
messageCount: messageElements.length,
|
||||
messageText,
|
||||
modelLabel: modelTrigger?.textContent?.trim() ?? '',
|
||||
composerValue: inputValue(state, '#prompt-textarea'),
|
||||
composerFocused: isFocused(state, '#prompt-textarea'),
|
||||
emailValue: inputValue(state, '#email'),
|
||||
emailFocused: isFocused(state, '#email'),
|
||||
passwordValue: inputValue(state, '#password'),
|
||||
passwordFocused: isFocused(state, '#password'),
|
||||
questionCount: visibleTextCount(state, 'p', HITL_QUESTION),
|
||||
answerOptionCount: visibleTextCount(state, 'button', HITL_OPTION, true),
|
||||
finalReplyCount: messageElements.filter((element) =>
|
||||
(element.textContent ?? '').includes(FINAL_REPLY),
|
||||
).length,
|
||||
completedAnswerCount: messageElements.filter((element) =>
|
||||
(element.textContent ?? '').includes(COMPLETED_ANSWER),
|
||||
).length,
|
||||
isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
|
||||
hasComposer: state.document.querySelector('#prompt-textarea') !== null,
|
||||
loginEmail: target(state, '#email', 'Login email'),
|
||||
loginPassword: target(state, '#password', 'Login password'),
|
||||
loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
|
||||
composer: target(state, '#prompt-textarea', 'Message input'),
|
||||
modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
|
||||
hitlModelSpec: target(state, '[role="option"]', HITL_MODEL_SPEC, HITL_MODEL_SPEC),
|
||||
stagingOption: target(state, 'button', 'Answer Staging', HITL_OPTION, true),
|
||||
};
|
||||
});
|
||||
|
||||
export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
|
||||
|
||||
export const hitlLifecycleActions = actions((): Action[] => {
|
||||
const state = ui.current;
|
||||
|
||||
if (state.path === '/login') {
|
||||
reloadIssued = false;
|
||||
pausedReloadIssued = false;
|
||||
if (!state.emailFocused && state.emailValue === '') {
|
||||
return clickOrWait(state.loginEmail);
|
||||
}
|
||||
if (state.emailFocused && state.emailValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.passwordFocused && state.passwordValue === '') {
|
||||
return clickOrWait(state.loginPassword);
|
||||
}
|
||||
if (state.passwordFocused && state.passwordValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.loginSubmit);
|
||||
}
|
||||
|
||||
if (state.stagingOption) {
|
||||
if (!pausedReloadIssued) {
|
||||
pausedReloadIssued = true;
|
||||
return ['Reload'];
|
||||
}
|
||||
return clickOrWait(state.stagingOption);
|
||||
}
|
||||
|
||||
if (state.isSubmitting || !state.hasComposer) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.finalReplyCount === 1) {
|
||||
if (!reloadIssued) {
|
||||
reloadIssued = true;
|
||||
return ['Reload'];
|
||||
}
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
|
||||
if (isPersistedConversation && state.messageCount === 0) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.messageCount === 0) {
|
||||
if (state.modelLabel !== HITL_MODEL_SPEC) {
|
||||
return state.hitlModelSpec
|
||||
? clickOrWait(state.hitlModelSpec)
|
||||
: clickOrWait(state.modelTrigger);
|
||||
}
|
||||
if (state.composerValue === '') {
|
||||
return state.composerFocused
|
||||
? [{ TypeText: { text: HITL_PROMPT, delayMillis: 0 } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
return state.composerFocused
|
||||
? [{ PressKey: { code: ENTER_KEY_CODE } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
|
||||
return ['Wait'];
|
||||
});
|
||||
|
||||
/** The run must reach a real, answerable ask_user_question pause, including after reload. */
|
||||
export const hitlQuestionEventuallyPauses = eventually(
|
||||
() => ui.current.questionCount === 1 && ui.current.answerOptionCount === 1,
|
||||
).within(25, 'seconds');
|
||||
|
||||
/** Answering resumes the checkpointed run and produces one terminal reply. */
|
||||
export const hitlAnswerEventuallyResumes = eventually(
|
||||
() =>
|
||||
ui.current.finalReplyCount === 1 &&
|
||||
ui.current.completedAnswerCount === 1 &&
|
||||
ui.current.answerOptionCount === 0,
|
||||
).within(40, 'seconds');
|
||||
|
||||
/** Duplicate cards or duplicate resume completions indicate a broken pause lifecycle. */
|
||||
export const hitlPauseAndResumeStaySingular = always(
|
||||
() =>
|
||||
ui.current.questionCount <= 1 &&
|
||||
ui.current.answerOptionCount <= 1 &&
|
||||
ui.current.finalReplyCount <= 1 &&
|
||||
ui.current.completedAnswerCount <= 1 &&
|
||||
ui.current.messageCount <= 2,
|
||||
);
|
||||
|
||||
/** After reload, the question remains an audit record without becoming answerable again. */
|
||||
export const answeredHitlStateSurvivesReload = always(() =>
|
||||
now(() => ui.current.lastAction === 'Reload').implies(
|
||||
eventually(
|
||||
() =>
|
||||
ui.current.finalReplyCount === 1 &&
|
||||
ui.current.completedAnswerCount === 1 &&
|
||||
ui.current.questionCount === 1 &&
|
||||
ui.current.answerOptionCount === 0,
|
||||
).within(20, 'seconds'),
|
||||
),
|
||||
);
|
||||
211
e2e/bombadil/model-lifecycle.specification.ts
Normal file
211
e2e/bombadil/model-lifecycle.specification.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
import { always, eventually, extract, now } from '@antithesishq/bombadil';
|
||||
import { actions } from '@antithesishq/bombadil/browser';
|
||||
import type { Action, Point, State } from '@antithesishq/bombadil/browser';
|
||||
import {
|
||||
noConsoleErrors,
|
||||
noHttpErrorCodes,
|
||||
noUncaughtExceptions,
|
||||
noUnhandledPromiseRejections,
|
||||
} from '@antithesishq/bombadil/browser/defaults/properties';
|
||||
|
||||
type Target = {
|
||||
name: string;
|
||||
point: Point;
|
||||
};
|
||||
|
||||
const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
const ENTER_KEY_CODE = 13;
|
||||
const TARGET_MODEL_SPEC = 'E2E Starters';
|
||||
const INITIAL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-initial';
|
||||
const SELECTED_MODEL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-selected';
|
||||
let reloadIssued = false;
|
||||
|
||||
function visiblePoint(state: State, element: Element | null): Point | null {
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
const style = state.window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.pointerEvents === 'none' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const point = {
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
};
|
||||
if (
|
||||
point.x < 0 ||
|
||||
point.y < 0 ||
|
||||
point.x > state.window.innerWidth ||
|
||||
point.y > state.window.innerHeight
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const hitElement = state.document.elementFromPoint(point.x, point.y);
|
||||
if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
|
||||
return null;
|
||||
}
|
||||
return point;
|
||||
}
|
||||
|
||||
function target(state: State, selector: string, name: string, text?: string): Target | null {
|
||||
for (const element of state.document.querySelectorAll(selector)) {
|
||||
if (text != null && element.textContent?.trim() !== text) {
|
||||
continue;
|
||||
}
|
||||
if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
|
||||
continue;
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
if (point) {
|
||||
return { name, point };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function inputValue(state: State, selector: string): string {
|
||||
const element = state.document.querySelector(selector) as
|
||||
| HTMLInputElement
|
||||
| HTMLTextAreaElement
|
||||
| null;
|
||||
return element?.value ?? '';
|
||||
}
|
||||
|
||||
function isFocused(state: State, selector: string): boolean {
|
||||
return state.document.activeElement?.matches(selector) === true;
|
||||
}
|
||||
|
||||
function clickOrWait(targetValue: Target | null): Action[] {
|
||||
return targetValue ? [{ Click: targetValue }] : ['Wait'];
|
||||
}
|
||||
|
||||
const ui = extract((state: State) => {
|
||||
const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
|
||||
return {
|
||||
path: state.window.location.pathname,
|
||||
lastAction: state.lastAction,
|
||||
messageCount: state.document.querySelectorAll('.message-render').length,
|
||||
composerValue: inputValue(state, '#prompt-textarea'),
|
||||
composerFocused: isFocused(state, '#prompt-textarea'),
|
||||
emailValue: inputValue(state, '#email'),
|
||||
emailFocused: isFocused(state, '#email'),
|
||||
passwordValue: inputValue(state, '#password'),
|
||||
passwordFocused: isFocused(state, '#password'),
|
||||
modelLabel: modelTrigger?.textContent?.trim() ?? '',
|
||||
isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
|
||||
hasComposer: state.document.querySelector('#prompt-textarea') !== null,
|
||||
loginEmail: target(state, '#email', 'Login email'),
|
||||
loginPassword: target(state, '#password', 'Login password'),
|
||||
loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
|
||||
composer: target(state, '#prompt-textarea', 'Message input'),
|
||||
modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
|
||||
starterSpec: target(state, '[role="option"]', TARGET_MODEL_SPEC, TARGET_MODEL_SPEC),
|
||||
};
|
||||
});
|
||||
|
||||
export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
|
||||
|
||||
export const modelLifecycleActions = actions((): Action[] => {
|
||||
const state = ui.current;
|
||||
|
||||
if (state.path === '/login') {
|
||||
reloadIssued = false;
|
||||
if (!state.emailFocused && state.emailValue === '') {
|
||||
return clickOrWait(state.loginEmail);
|
||||
}
|
||||
if (state.emailFocused && state.emailValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.passwordFocused && state.passwordValue === '') {
|
||||
return clickOrWait(state.loginPassword);
|
||||
}
|
||||
if (state.passwordFocused && state.passwordValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.loginSubmit);
|
||||
}
|
||||
|
||||
if (state.isSubmitting || !state.hasComposer) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
|
||||
if (isPersistedConversation && state.messageCount === 0) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.messageCount === 0) {
|
||||
if (state.composerValue === '') {
|
||||
if (state.composerFocused) {
|
||||
return [{ TypeText: { text: INITIAL_PROMPT, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
|
||||
if (state.messageCount === 2) {
|
||||
if (state.modelLabel !== TARGET_MODEL_SPEC) {
|
||||
if (state.starterSpec) {
|
||||
return clickOrWait(state.starterSpec);
|
||||
}
|
||||
return clickOrWait(state.modelTrigger);
|
||||
}
|
||||
if (state.composerValue === '') {
|
||||
if (state.composerFocused) {
|
||||
return [{ TypeText: { text: SELECTED_MODEL_PROMPT, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
|
||||
if (state.messageCount >= 4 && state.modelLabel === TARGET_MODEL_SPEC) {
|
||||
if (!reloadIssued) {
|
||||
reloadIssued = true;
|
||||
return ['Reload'];
|
||||
}
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
return ['Wait'];
|
||||
});
|
||||
|
||||
/**
|
||||
* A model choice can remain local until it is used for a submission. Once a turn
|
||||
* has been sent with that model, both the selection and message history must be
|
||||
* server-backed and recoverable after reload.
|
||||
*/
|
||||
export const submittedModelAndMessagesSurviveReload = always(
|
||||
() =>
|
||||
ui.current.messageCount < 4 ||
|
||||
ui.current.modelLabel === '' ||
|
||||
ui.current.modelLabel === TARGET_MODEL_SPEC,
|
||||
);
|
||||
|
||||
export const selectedModelExchangeEventuallyCommits = eventually(
|
||||
() => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC,
|
||||
).within(25, 'seconds');
|
||||
|
||||
/** The committed model selection and all four turns must rehydrate after reload. */
|
||||
export const selectedModelExchangeSurvivesReload = always(() =>
|
||||
now(() => ui.current.lastAction === 'Reload').implies(
|
||||
eventually(
|
||||
() => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC,
|
||||
).within(20, 'seconds'),
|
||||
),
|
||||
);
|
||||
570
e2e/bombadil/specification.ts
Normal file
570
e2e/bombadil/specification.ts
Normal file
|
|
@ -0,0 +1,570 @@
|
|||
import { always, eventually, extract, from, integers, next, now } from '@antithesishq/bombadil';
|
||||
import { actions } from '@antithesishq/bombadil/browser';
|
||||
import type { Action, Point, State } from '@antithesishq/bombadil/browser';
|
||||
import {
|
||||
noConsoleErrors,
|
||||
noHttpErrorCodes,
|
||||
noUncaughtExceptions,
|
||||
noUnhandledPromiseRejections,
|
||||
} from '@antithesishq/bombadil/browser/defaults/properties';
|
||||
|
||||
type Target = {
|
||||
name: string;
|
||||
point: Point;
|
||||
};
|
||||
|
||||
type NavigationStatus = {
|
||||
current: number;
|
||||
total: number;
|
||||
previousDisabled: boolean;
|
||||
nextDisabled: boolean;
|
||||
};
|
||||
|
||||
const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
const ENTER_KEY_CODE = 13;
|
||||
const RENAME_MARKER_PREFIX = ' — Bombadil:';
|
||||
const MAX_CONVERSATION_FINGERPRINT_LENGTH = 80;
|
||||
const PROMPT_VARIANTS = from([
|
||||
{ name: 'short', suffix: '' },
|
||||
{ name: 'unicode-雪-🙂', suffix: '' },
|
||||
{ name: 'spaces', suffix: ' with internal spaces' },
|
||||
{ name: 'markdown', suffix: ' [x](y)' },
|
||||
{ name: 'long', suffix: `-${'x'.repeat(256)}` },
|
||||
]);
|
||||
const PROMPT_NONCES = integers().min(0).max(2_147_483_647);
|
||||
|
||||
function generatePrompt(): string {
|
||||
const variant = PROMPT_VARIANTS.generate();
|
||||
return `E2E_REPLY:bombadil-${variant.name}-${PROMPT_NONCES.generate()}${variant.suffix}`;
|
||||
}
|
||||
|
||||
function promptMarker(text: string): string {
|
||||
return text.match(/E2E_REPLY:[^\s]+/)?.[0] ?? '';
|
||||
}
|
||||
|
||||
function conversationFingerprint(text: string): string {
|
||||
return promptMarker(text).slice(0, MAX_CONVERSATION_FINGERPRINT_LENGTH);
|
||||
}
|
||||
|
||||
function expectedReply(text: string): string {
|
||||
const marker = promptMarker(text);
|
||||
return marker === '' ? '' : `E2E reply ${marker.slice('E2E_REPLY:'.length)}`;
|
||||
}
|
||||
|
||||
function visiblePoint(state: State, element: Element | null): Point | null {
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
const style = state.window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.pointerEvents === 'none' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const point = {
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
};
|
||||
if (
|
||||
point.x < 0 ||
|
||||
point.y < 0 ||
|
||||
point.x > state.window.innerWidth ||
|
||||
point.y > state.window.innerHeight
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const hitElement = state.document.elementFromPoint(point.x, point.y);
|
||||
if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
|
||||
return null;
|
||||
}
|
||||
return point;
|
||||
}
|
||||
|
||||
function target(
|
||||
state: State,
|
||||
selector: string,
|
||||
name: string,
|
||||
text?: string,
|
||||
last = false,
|
||||
): Target | null {
|
||||
const candidates = Array.from(state.document.querySelectorAll(selector)).filter((element) => {
|
||||
if (text == null) {
|
||||
return true;
|
||||
}
|
||||
return element.textContent?.trim() === text;
|
||||
});
|
||||
const elements = last ? candidates.reverse() : candidates;
|
||||
for (const element of elements) {
|
||||
if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
|
||||
continue;
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
if (point) {
|
||||
return { name, point };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function conversationTargets(state: State): Target[] {
|
||||
return Array.from(state.document.querySelectorAll('[data-testid="convo-item"]')).flatMap(
|
||||
(element, index) => {
|
||||
if (element.querySelector('[aria-current="page"]')) {
|
||||
return [];
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
return point ? [{ name: `Open conversation ${index + 1}`, point }] : [];
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function clickedConversationIndex(lastAction: unknown): number | null {
|
||||
if (typeof lastAction !== 'object' || lastAction === null || !('Click' in lastAction)) {
|
||||
return null;
|
||||
}
|
||||
const click = lastAction.Click;
|
||||
if (typeof click !== 'object' || click === null || !('name' in click)) {
|
||||
return null;
|
||||
}
|
||||
const match = String(click.name).match(/^Open conversation (\d+)$/);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
function isPersistedConversation(pathname: string): boolean {
|
||||
return pathname.startsWith('/c/') && pathname !== '/c/new';
|
||||
}
|
||||
|
||||
function inputValue(state: State, selector: string): string {
|
||||
const element = state.document.querySelector(selector) as
|
||||
| HTMLInputElement
|
||||
| HTMLTextAreaElement
|
||||
| null;
|
||||
return element?.value ?? '';
|
||||
}
|
||||
|
||||
function isFocused(state: State, selector: string): boolean {
|
||||
return state.document.activeElement?.matches(selector) === true;
|
||||
}
|
||||
|
||||
function clickAction(targetValue: Target | null): Action[] {
|
||||
return targetValue ? [{ Click: targetValue }] : [];
|
||||
}
|
||||
|
||||
function clickOrWait(targetValue: Target | null): Action[] {
|
||||
const clicks = clickAction(targetValue);
|
||||
return clicks.length > 0 ? clicks : ['Wait'];
|
||||
}
|
||||
|
||||
const ui = extract((state: State) => {
|
||||
const statuses: NavigationStatus[] = [];
|
||||
for (const navigation of state.document.querySelectorAll(
|
||||
'nav[aria-label="Sibling message navigation"]',
|
||||
)) {
|
||||
const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? '';
|
||||
const match = text.match(/^(\d+)\s*\/\s*(\d+)$/);
|
||||
const previous = navigation.querySelector(
|
||||
'button[aria-label="Previous sibling message"]',
|
||||
) as HTMLButtonElement | null;
|
||||
const nextButton = navigation.querySelector(
|
||||
'button[aria-label="Next sibling message"]',
|
||||
) as HTMLButtonElement | null;
|
||||
if (match && previous && nextButton) {
|
||||
statuses.push({
|
||||
current: Number(match[1]),
|
||||
total: Number(match[2]),
|
||||
previousDisabled: previous.disabled || previous.getAttribute('aria-disabled') === 'true',
|
||||
nextDisabled: nextButton.disabled || nextButton.getAttribute('aria-disabled') === 'true',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const messageIds = Array.from(state.document.querySelectorAll('.message-render'))
|
||||
.map((element) => element.id)
|
||||
.filter(Boolean);
|
||||
const messageText = Array.from(state.document.querySelectorAll('.message-render'))
|
||||
.map((element) => element.textContent ?? '')
|
||||
.join('\n');
|
||||
const parallelColumnCounts = Array.from(
|
||||
state.document.querySelectorAll('.sibling-content-group'),
|
||||
).map((group) => group.children.length);
|
||||
const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
|
||||
const conversationElements = Array.from(
|
||||
state.document.querySelectorAll('[data-testid="convo-item"]'),
|
||||
);
|
||||
const activeConversationIndexes = conversationElements.flatMap((element, index) =>
|
||||
element.querySelector('[aria-current="page"]') ? [index + 1] : [],
|
||||
);
|
||||
const activeConversation = conversationElements.find((element) =>
|
||||
element.querySelector('[aria-current="page"]'),
|
||||
);
|
||||
|
||||
return {
|
||||
path: state.window.location.pathname,
|
||||
lastAction: state.lastAction,
|
||||
messageText,
|
||||
messageIds,
|
||||
statuses,
|
||||
parallelColumnCounts,
|
||||
composerValue: inputValue(state, '#prompt-textarea'),
|
||||
composerFocused: isFocused(state, '#prompt-textarea'),
|
||||
renameValue: inputValue(state, 'input[aria-label="New Conversation Title"]'),
|
||||
renameFocused: isFocused(state, 'input[aria-label="New Conversation Title"]'),
|
||||
emailValue: inputValue(state, '#email'),
|
||||
emailFocused: isFocused(state, '#email'),
|
||||
passwordValue: inputValue(state, '#password'),
|
||||
passwordFocused: isFocused(state, '#password'),
|
||||
modelLabel: modelTrigger?.textContent?.trim() ?? '',
|
||||
activeConversationIndexes,
|
||||
activeConversationTitle: activeConversation?.textContent?.trim() ?? '',
|
||||
modelOptionsOpen: state.document.querySelector('[role="option"]') !== null,
|
||||
hasAddedConversation:
|
||||
state.document.querySelector('button[aria-label="Close added conversation"]') !== null,
|
||||
isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
|
||||
hasComposer: state.document.querySelector('#prompt-textarea') !== null,
|
||||
loginEmail: target(state, '#email', 'Login email'),
|
||||
loginPassword: target(state, '#password', 'Login password'),
|
||||
loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
|
||||
modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
|
||||
providerA: target(state, '[role="option"]', 'Mock Provider A', 'Mock Provider A'),
|
||||
providerB: target(state, '[role="option"]', 'Mock Provider B', 'Mock Provider B'),
|
||||
modelA: target(state, '[role="option"]', 'mock-model-a', 'mock-model-a'),
|
||||
modelB: target(state, '[role="option"]', 'mock-model-b', 'mock-model-b'),
|
||||
starterSpec: target(state, '[role="option"]', 'E2E Starters', 'E2E Starters'),
|
||||
composer: target(state, '#prompt-textarea', 'Message input'),
|
||||
addMultiConversation: target(
|
||||
state,
|
||||
'[data-testid="add-multi-convo-button"]',
|
||||
'Add multi-conversation',
|
||||
),
|
||||
closeAddedConversation: target(
|
||||
state,
|
||||
'button[aria-label="Close added conversation"]',
|
||||
'Close added conversation',
|
||||
),
|
||||
regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', undefined, true),
|
||||
previousSibling: target(
|
||||
state,
|
||||
'button[aria-label="Previous sibling message"]',
|
||||
'Previous sibling message',
|
||||
undefined,
|
||||
true,
|
||||
),
|
||||
nextSibling: target(
|
||||
state,
|
||||
'button[aria-label="Next sibling message"]',
|
||||
'Next sibling message',
|
||||
undefined,
|
||||
true,
|
||||
),
|
||||
branchParallel: target(
|
||||
state,
|
||||
'button[aria-label="Create branch from this response"]',
|
||||
'Create branch from parallel response',
|
||||
undefined,
|
||||
true,
|
||||
),
|
||||
newConversation: target(state, '[data-testid="new-chat-button"]', 'New conversation'),
|
||||
conversationItems: conversationTargets(state),
|
||||
conversationMenu: target(
|
||||
state,
|
||||
'button[aria-label="Conversation Menu Options"]',
|
||||
'Conversation menu',
|
||||
),
|
||||
renameMenuItem: target(state, '[role="menuitem"]', 'Rename conversation', 'Rename'),
|
||||
renameInput: target(state, 'input[aria-label="New Conversation Title"]', 'Conversation title'),
|
||||
renameSave: target(state, 'button[aria-label="Save"]', 'Save conversation title'),
|
||||
};
|
||||
});
|
||||
|
||||
export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
|
||||
|
||||
export const libreChatActions = actions(() => {
|
||||
const state = ui.current;
|
||||
|
||||
if (state.path === '/login') {
|
||||
if (!state.emailFocused && state.emailValue === '') {
|
||||
return clickOrWait(state.loginEmail);
|
||||
}
|
||||
if (state.emailFocused && state.emailValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.passwordFocused && state.passwordValue === '') {
|
||||
return clickOrWait(state.loginPassword);
|
||||
}
|
||||
if (state.passwordFocused && state.passwordValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.loginSubmit);
|
||||
}
|
||||
|
||||
if (state.renameInput) {
|
||||
if (!state.renameFocused) {
|
||||
return clickAction(state.renameInput);
|
||||
}
|
||||
if (!state.renameValue.includes(RENAME_MARKER_PREFIX.trim())) {
|
||||
const fingerprint = conversationFingerprint(state.messageText);
|
||||
return fingerprint === ''
|
||||
? ['Wait']
|
||||
: [{ TypeText: { text: `${RENAME_MARKER_PREFIX}${fingerprint}`, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.renameSave);
|
||||
}
|
||||
|
||||
if (state.isSubmitting) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const selectorActions: Action[] = ['Wait'];
|
||||
if (state.providerA) {
|
||||
selectorActions.push(...clickAction(state.providerA));
|
||||
}
|
||||
if (state.providerB) {
|
||||
selectorActions.push(...clickAction(state.providerB));
|
||||
}
|
||||
if (state.modelA) {
|
||||
selectorActions.push(...clickAction(state.modelA));
|
||||
}
|
||||
if (state.modelB) {
|
||||
selectorActions.push(...clickAction(state.modelB));
|
||||
}
|
||||
if (state.starterSpec) {
|
||||
selectorActions.push(...clickAction(state.starterSpec));
|
||||
}
|
||||
if (state.modelOptionsOpen || selectorActions.length > 1) {
|
||||
if (state.modelOptionsOpen) {
|
||||
// Clicking a visible point outside the popover closes unknown submenu
|
||||
// states without getting trapped in a keyboard-action retry loop.
|
||||
selectorActions.push(...clickAction(state.composer));
|
||||
}
|
||||
return selectorActions;
|
||||
}
|
||||
|
||||
if (state.renameMenuItem) {
|
||||
return ['Wait', ...clickAction(state.renameMenuItem)];
|
||||
}
|
||||
|
||||
const isPersisted = isPersistedConversation(state.path);
|
||||
if (!state.hasComposer || (isPersisted && state.messageIds.length === 0)) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const possible: Action[] = ['Wait'];
|
||||
if (state.modelLabel === '' || state.modelLabel === 'Select a model') {
|
||||
possible.push(...clickAction(state.modelTrigger));
|
||||
return possible;
|
||||
}
|
||||
|
||||
const composeActions = (): Action[] => {
|
||||
if (state.composerValue !== '') {
|
||||
if (state.composerFocused) {
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
if (state.composerFocused) {
|
||||
return [{ TypeText: { text: generatePrompt(), delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.composer);
|
||||
};
|
||||
|
||||
if (state.messageIds.length === 0) {
|
||||
return composeActions();
|
||||
}
|
||||
|
||||
if (state.parallelColumnCounts.length === 0) {
|
||||
if (!state.hasAddedConversation) {
|
||||
return clickOrWait(state.addMultiConversation);
|
||||
}
|
||||
return composeActions();
|
||||
}
|
||||
|
||||
if (state.statuses.length === 0 && state.branchParallel) {
|
||||
return clickAction(state.branchParallel);
|
||||
}
|
||||
if (state.composerValue === '') {
|
||||
if (state.composerFocused) {
|
||||
possible.push({ TypeText: { text: generatePrompt(), delayMillis: 0 } });
|
||||
} else {
|
||||
possible.push(...clickAction(state.composer));
|
||||
}
|
||||
} else if (state.composerFocused) {
|
||||
possible.push({ PressKey: { code: ENTER_KEY_CODE } });
|
||||
}
|
||||
|
||||
possible.push(...clickAction(state.modelTrigger));
|
||||
possible.push(...clickAction(state.addMultiConversation));
|
||||
possible.push(...clickAction(state.closeAddedConversation));
|
||||
possible.push(...clickAction(state.regenerate));
|
||||
possible.push(...clickAction(state.previousSibling));
|
||||
possible.push(...clickAction(state.nextSibling));
|
||||
possible.push(...clickAction(state.branchParallel));
|
||||
possible.push(...clickAction(state.newConversation));
|
||||
possible.push(...state.conversationItems.map((item) => ({ Click: item }) as Action));
|
||||
possible.push(...clickAction(state.conversationMenu));
|
||||
possible.push(...clickAction(state.renameMenuItem));
|
||||
|
||||
if (state.path.startsWith('/c/') && state.path !== '/c/new') {
|
||||
possible.push('Reload');
|
||||
}
|
||||
return possible;
|
||||
});
|
||||
|
||||
export const messageIdsRemainUnique = always(() => {
|
||||
const ids = ui.current.messageIds;
|
||||
return new Set(ids).size === ids.length;
|
||||
});
|
||||
|
||||
export const siblingNavigationRemainsValid = always(() =>
|
||||
ui.current.statuses.every(
|
||||
({ current, total, previousDisabled, nextDisabled }) =>
|
||||
total > 1 &&
|
||||
current >= 1 &&
|
||||
current <= total &&
|
||||
previousDisabled === (current === 1) &&
|
||||
nextDisabled === (current === total),
|
||||
),
|
||||
);
|
||||
|
||||
export const multiConversationAlwaysRendersTwoColumns = always(() =>
|
||||
ui.current.parallelColumnCounts.every((count) => count === 2),
|
||||
);
|
||||
|
||||
export const sidebarNavigationEventuallySelectsTarget = always(() => {
|
||||
const expectedIndex = clickedConversationIndex(ui.current.lastAction);
|
||||
return now(() => expectedIndex !== null).implies(
|
||||
eventually(
|
||||
() =>
|
||||
expectedIndex !== null &&
|
||||
isPersistedConversation(ui.current.path) &&
|
||||
ui.current.activeConversationIndexes.length === 1 &&
|
||||
ui.current.activeConversationIndexes[0] === expectedIndex &&
|
||||
ui.current.messageIds.length > 0,
|
||||
).within(10, 'seconds'),
|
||||
);
|
||||
});
|
||||
|
||||
export const sidebarEventuallyMatchesRenderedConversation = always(() =>
|
||||
now(
|
||||
() =>
|
||||
isPersistedConversation(ui.current.path) &&
|
||||
ui.current.hasComposer &&
|
||||
ui.current.messageIds.length > 0,
|
||||
).implies(
|
||||
eventually(() => {
|
||||
const state = ui.current;
|
||||
const fingerprint = conversationFingerprint(state.activeConversationTitle);
|
||||
return (
|
||||
isPersistedConversation(state.path) &&
|
||||
state.activeConversationIndexes.length === 1 &&
|
||||
(fingerprint === '' || state.messageText.includes(fingerprint))
|
||||
);
|
||||
}).within(10, 'seconds'),
|
||||
),
|
||||
);
|
||||
|
||||
export const loginEventuallySucceeds = eventually(() => ui.current.path !== '/login').within(
|
||||
30,
|
||||
'seconds',
|
||||
);
|
||||
|
||||
export const anExchangeEventuallyHappens = eventually(
|
||||
() => ui.current.messageIds.length >= 2,
|
||||
).within(60, 'seconds');
|
||||
|
||||
export const multiConversationEventuallyRenders = eventually(
|
||||
() => ui.current.parallelColumnCounts.length > 0,
|
||||
).within(75, 'seconds');
|
||||
|
||||
export const aBranchEventuallyRenders = eventually(() => ui.current.statuses.length > 0).within(
|
||||
85,
|
||||
'seconds',
|
||||
);
|
||||
|
||||
export const streamingEventuallyTerminates = always(() =>
|
||||
now(() => ui.current.isSubmitting).implies(
|
||||
eventually(() => !ui.current.isSubmitting).within(45, 'seconds'),
|
||||
),
|
||||
);
|
||||
|
||||
export const submittedPromptEventuallyAppears = always(() => {
|
||||
const submittedText = ui.current.composerValue.trim();
|
||||
const marker = promptMarker(submittedText);
|
||||
const eligible = ui.current.composerFocused && submittedText !== '';
|
||||
return next(
|
||||
now(
|
||||
() =>
|
||||
eligible &&
|
||||
typeof ui.current.lastAction === 'object' &&
|
||||
ui.current.lastAction !== null &&
|
||||
'PressKey' in ui.current.lastAction &&
|
||||
ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
|
||||
).implies(
|
||||
eventually(() => marker !== '' && ui.current.messageText.includes(marker)).within(
|
||||
30,
|
||||
'seconds',
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
export const assistantReplyEventuallyAppears = always(() => {
|
||||
const replyText = expectedReply(ui.current.composerValue.trim());
|
||||
const eligible = ui.current.composerFocused && replyText !== '';
|
||||
return next(
|
||||
now(
|
||||
() =>
|
||||
eligible &&
|
||||
typeof ui.current.lastAction === 'object' &&
|
||||
ui.current.lastAction !== null &&
|
||||
'PressKey' in ui.current.lastAction &&
|
||||
ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
|
||||
).implies(eventually(() => ui.current.messageText.includes(replyText)).within(30, 'seconds')),
|
||||
);
|
||||
});
|
||||
|
||||
export const composerEventuallyClearsAfterSubmit = always(() => {
|
||||
const eligible = ui.current.composerFocused && ui.current.composerValue.trim() !== '';
|
||||
return next(
|
||||
now(
|
||||
() =>
|
||||
eligible &&
|
||||
typeof ui.current.lastAction === 'object' &&
|
||||
ui.current.lastAction !== null &&
|
||||
'PressKey' in ui.current.lastAction &&
|
||||
ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
|
||||
).implies(eventually(() => ui.current.composerValue === '').within(5, 'seconds')),
|
||||
);
|
||||
});
|
||||
|
||||
export const multiConversationSubmissionEventuallyRendersInParallel = always(() => {
|
||||
const submittedText = ui.current.composerValue.trim();
|
||||
const marker = promptMarker(submittedText);
|
||||
const eligible =
|
||||
ui.current.hasAddedConversation && ui.current.composerFocused && submittedText !== '';
|
||||
return next(
|
||||
now(
|
||||
() =>
|
||||
eligible &&
|
||||
typeof ui.current.lastAction === 'object' &&
|
||||
ui.current.lastAction !== null &&
|
||||
'PressKey' in ui.current.lastAction &&
|
||||
ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
|
||||
).implies(
|
||||
eventually(() => {
|
||||
const counts = ui.current.parallelColumnCounts;
|
||||
return (
|
||||
marker !== '' &&
|
||||
ui.current.messageText.includes(marker) &&
|
||||
counts.length > 0 &&
|
||||
counts[counts.length - 1] === 2
|
||||
);
|
||||
}).within(30, 'seconds'),
|
||||
),
|
||||
);
|
||||
});
|
||||
317
e2e/bombadil/steering-lifecycle.specification.ts
Normal file
317
e2e/bombadil/steering-lifecycle.specification.ts
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
import { always, eventually, extract, now } from '@antithesishq/bombadil';
|
||||
import { actions } from '@antithesishq/bombadil/browser';
|
||||
import type { Action, Point, State } from '@antithesishq/bombadil/browser';
|
||||
import {
|
||||
noConsoleErrors,
|
||||
noHttpErrorCodes,
|
||||
noUncaughtExceptions,
|
||||
noUnhandledPromiseRejections,
|
||||
} from '@antithesishq/bombadil/browser/defaults/properties';
|
||||
|
||||
type Target = {
|
||||
name: string;
|
||||
point: Point;
|
||||
};
|
||||
|
||||
const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
|
||||
const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
|
||||
const ENTER_KEY_CODE = 13;
|
||||
const ESCAPE_KEY_CODE = 27;
|
||||
const PROVIDER = 'Mock Provider C';
|
||||
const MODEL = 'mock-model-c';
|
||||
const MCP_SERVER = 'E2E Memory';
|
||||
const SETUP_PROMPT = 'E2E_REPLY:bombadil-steering-setup';
|
||||
const SETUP_REPLY = 'E2E reply bombadil-steering-setup';
|
||||
const STEER_LABEL = 'bombadil-steering';
|
||||
const STEER_PROMPT = `E2E_STEER_TOOL_REPLY:${STEER_LABEL}`;
|
||||
const STEER_TEXT = `Steer injection ${STEER_LABEL}`;
|
||||
const FINAL_REPLY = `E2E steer tool reply done ${STEER_LABEL}`;
|
||||
let reloadIssued = false;
|
||||
|
||||
function visiblePoint(state: State, element: Element | null): Point | null {
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
const style = state.window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (
|
||||
style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.pointerEvents === 'none' ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
|
||||
const hitElement = state.document.elementFromPoint(point.x, point.y);
|
||||
if (
|
||||
point.x < 0 ||
|
||||
point.y < 0 ||
|
||||
point.x > state.window.innerWidth ||
|
||||
point.y > state.window.innerHeight ||
|
||||
!hitElement ||
|
||||
(hitElement !== element && !element.contains(hitElement))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return point;
|
||||
}
|
||||
|
||||
function target(
|
||||
state: State,
|
||||
selector: string,
|
||||
name: string,
|
||||
text?: string,
|
||||
containsText = false,
|
||||
): Target | null {
|
||||
for (const element of state.document.querySelectorAll(selector)) {
|
||||
const content = element.textContent?.trim() ?? '';
|
||||
if (text != null && (containsText ? !content.includes(text) : content !== text)) {
|
||||
continue;
|
||||
}
|
||||
if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
|
||||
continue;
|
||||
}
|
||||
const point = visiblePoint(state, element);
|
||||
if (point) {
|
||||
return { name, point };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function inputValue(state: State, selector: string): string {
|
||||
return (
|
||||
state.document.querySelector<HTMLInputElement | HTMLTextAreaElement>(selector)?.value ?? ''
|
||||
);
|
||||
}
|
||||
|
||||
function isFocused(state: State, selector: string): boolean {
|
||||
return state.document.activeElement?.matches(selector) === true;
|
||||
}
|
||||
|
||||
function clickOrWait(targetValue: Target | null): Action[] {
|
||||
return targetValue ? [{ Click: targetValue }] : ['Wait'];
|
||||
}
|
||||
|
||||
const ui = extract((state: State) => {
|
||||
const messageElements = Array.from(state.document.querySelectorAll('.message-render'));
|
||||
const messageText = messageElements.map((element) => element.textContent ?? '').join('\n');
|
||||
const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
|
||||
const duringRunSend = state.document.querySelector('[data-testid="during-run-send-button"]');
|
||||
const mcpServerItem = Array.from(
|
||||
state.document.querySelectorAll('[role="menuitemcheckbox"]'),
|
||||
).find((element) => (element.textContent ?? '').includes(MCP_SERVER));
|
||||
const selectedMcpButton = Array.from(
|
||||
state.document.querySelectorAll('button[aria-expanded]'),
|
||||
).find((element) => (element.textContent ?? '').includes(MCP_SERVER));
|
||||
return {
|
||||
path: state.window.location.pathname,
|
||||
lastAction: state.lastAction,
|
||||
lastActionWasEnter:
|
||||
typeof state.lastAction === 'object' &&
|
||||
state.lastAction !== null &&
|
||||
'PressKey' in state.lastAction &&
|
||||
state.lastAction.PressKey.code === ENTER_KEY_CODE,
|
||||
messageCount: messageElements.length,
|
||||
messageText,
|
||||
modelLabel: modelTrigger?.textContent?.trim() ?? '',
|
||||
composerValue: inputValue(state, '#prompt-textarea'),
|
||||
composerFocused: isFocused(state, '#prompt-textarea'),
|
||||
emailValue: inputValue(state, '#email'),
|
||||
emailFocused: isFocused(state, '#email'),
|
||||
passwordValue: inputValue(state, '#password'),
|
||||
passwordFocused: isFocused(state, '#password'),
|
||||
mcpSelected:
|
||||
mcpServerItem?.getAttribute('aria-checked') === 'true' || selectedMcpButton !== undefined,
|
||||
mcpMenuOpen: visiblePoint(state, mcpServerItem ?? null) !== null,
|
||||
inFlightSteerCount: state.document.querySelectorAll('[data-testid="in-flight-steer"]').length,
|
||||
appliedSteerCount: state.document.querySelectorAll('[data-testid="steer-part"]').length,
|
||||
queuedMessageCount: state.document.querySelectorAll('[data-testid="queued-message-row"]')
|
||||
.length,
|
||||
finalReplyCount: messageElements.filter((element) =>
|
||||
(element.textContent ?? '').includes(FINAL_REPLY),
|
||||
).length,
|
||||
setupComplete: messageText.includes(SETUP_REPLY),
|
||||
duringRunAction: duringRunSend?.getAttribute('data-during-run-action') ?? '',
|
||||
isSubmitting:
|
||||
state.document.querySelector('[data-testid="stop-generation-button"]') !== null ||
|
||||
duringRunSend !== null,
|
||||
hasComposer: state.document.querySelector('#prompt-textarea') !== null,
|
||||
loginEmail: target(state, '#email', 'Login email'),
|
||||
loginPassword: target(state, '#password', 'Login password'),
|
||||
loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
|
||||
composer: target(state, '#prompt-textarea', 'Message input'),
|
||||
modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
|
||||
provider: target(state, '[role="option"]', PROVIDER, PROVIDER),
|
||||
model: target(state, '[role="option"]', MODEL, MODEL),
|
||||
mcpTrigger: target(state, 'button', 'MCP Servers', 'MCP Servers'),
|
||||
mcpServer: target(state, '[role="menuitemcheckbox"]', `Select ${MCP_SERVER}`, MCP_SERVER, true),
|
||||
};
|
||||
});
|
||||
|
||||
export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
|
||||
|
||||
export const steeringLifecycleActions = actions((): Action[] => {
|
||||
const state = ui.current;
|
||||
|
||||
if (state.path === '/login') {
|
||||
reloadIssued = false;
|
||||
if (!state.emailFocused && state.emailValue === '') {
|
||||
return clickOrWait(state.loginEmail);
|
||||
}
|
||||
if (state.emailFocused && state.emailValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
|
||||
}
|
||||
if (!state.passwordFocused && state.passwordValue === '') {
|
||||
return clickOrWait(state.loginPassword);
|
||||
}
|
||||
if (state.passwordFocused && state.passwordValue === '') {
|
||||
return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
|
||||
}
|
||||
return clickOrWait(state.loginSubmit);
|
||||
}
|
||||
|
||||
if (!state.hasComposer) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.composerValue.endsWith(STEER_TEXT)) {
|
||||
if (state.duringRunAction !== 'steer') {
|
||||
return ['Wait'];
|
||||
}
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
|
||||
if (
|
||||
state.setupComplete &&
|
||||
state.lastActionWasEnter &&
|
||||
state.inFlightSteerCount === 0 &&
|
||||
!state.composerValue.endsWith(STEER_TEXT)
|
||||
) {
|
||||
return state.composerFocused
|
||||
? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
|
||||
if (state.isSubmitting) {
|
||||
if (!state.setupComplete) {
|
||||
return ['Wait'];
|
||||
}
|
||||
if (state.inFlightSteerCount === 1 || state.appliedSteerCount === 1) {
|
||||
return ['Wait'];
|
||||
}
|
||||
if (state.composerValue === '') {
|
||||
return state.composerFocused
|
||||
? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
if (state.composerValue === STEER_PROMPT) {
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
if (state.duringRunAction !== 'steer') {
|
||||
return ['Wait'];
|
||||
}
|
||||
return state.composerFocused
|
||||
? [{ PressKey: { code: ENTER_KEY_CODE } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
|
||||
if (state.finalReplyCount === 1 && state.appliedSteerCount === 1) {
|
||||
if (!reloadIssued) {
|
||||
reloadIssued = true;
|
||||
return ['Reload'];
|
||||
}
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
|
||||
if (isPersistedConversation && state.messageCount === 0) {
|
||||
return ['Wait'];
|
||||
}
|
||||
|
||||
if (state.messageCount === 0) {
|
||||
if (state.modelLabel === '' || state.modelLabel === 'Select a model') {
|
||||
return clickOrWait(state.modelTrigger);
|
||||
}
|
||||
if (state.model) {
|
||||
return clickOrWait(state.model);
|
||||
}
|
||||
if (state.provider) {
|
||||
return clickOrWait(state.provider);
|
||||
}
|
||||
if (!state.modelLabel.includes(MODEL) && !state.modelLabel.includes(PROVIDER)) {
|
||||
return clickOrWait(state.modelTrigger);
|
||||
}
|
||||
if (!state.mcpSelected) {
|
||||
return state.mcpServer ? clickOrWait(state.mcpServer) : clickOrWait(state.mcpTrigger);
|
||||
}
|
||||
if (state.mcpMenuOpen) {
|
||||
return [{ PressKey: { code: ESCAPE_KEY_CODE } }];
|
||||
}
|
||||
if (state.composerValue === '') {
|
||||
return state.composerFocused
|
||||
? [{ TypeText: { text: SETUP_PROMPT, delayMillis: 0 } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
return state.composerFocused
|
||||
? [{ PressKey: { code: ENTER_KEY_CODE } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
|
||||
if (state.messageCount === 2 && state.setupComplete) {
|
||||
if (state.composerValue === '') {
|
||||
return state.composerFocused
|
||||
? [{ TypeText: { text: STEER_PROMPT, delayMillis: 0 } }]
|
||||
: clickOrWait(state.composer);
|
||||
}
|
||||
if (!state.composerFocused) {
|
||||
return clickOrWait(state.composer);
|
||||
}
|
||||
return [{ PressKey: { code: ENTER_KEY_CODE } }];
|
||||
}
|
||||
|
||||
return ['Wait'];
|
||||
});
|
||||
|
||||
/** A submitted steer must be represented immediately while the run is active. */
|
||||
export const steerEventuallyBecomesInFlight = eventually(
|
||||
() => ui.current.inFlightSteerCount === 1,
|
||||
).within(30, 'seconds');
|
||||
|
||||
/** The steer drains at the MCP tool boundary and becomes durable in-thread state. */
|
||||
export const steerEventuallyApplies = eventually(
|
||||
() =>
|
||||
ui.current.appliedSteerCount === 1 &&
|
||||
ui.current.inFlightSteerCount === 0 &&
|
||||
ui.current.finalReplyCount === 1,
|
||||
).within(65, 'seconds');
|
||||
|
||||
/** A steer is neither duplicated nor degraded into a queued follow-up turn. */
|
||||
export const steerStaysSingularAndInBand = always(
|
||||
() =>
|
||||
ui.current.inFlightSteerCount <= 1 &&
|
||||
ui.current.appliedSteerCount <= 1 &&
|
||||
ui.current.finalReplyCount <= 1 &&
|
||||
ui.current.queuedMessageCount === 0 &&
|
||||
ui.current.messageCount <= 4,
|
||||
);
|
||||
|
||||
/** The applied steer and terminal response must survive a conversation reload. */
|
||||
export const appliedSteerSurvivesReload = always(() =>
|
||||
now(() => ui.current.lastAction === 'Reload').implies(
|
||||
eventually(
|
||||
() =>
|
||||
ui.current.appliedSteerCount === 1 &&
|
||||
ui.current.inFlightSteerCount === 0 &&
|
||||
ui.current.finalReplyCount === 1,
|
||||
).within(20, 'seconds'),
|
||||
),
|
||||
);
|
||||
|
|
@ -211,3 +211,13 @@ modelSpecs:
|
|||
preset:
|
||||
endpoint: 'Mock Provider A'
|
||||
model: 'mock-model-a'
|
||||
|
||||
# Focused Bombadil HITL fixture. The fake model calls ask_user_question
|
||||
# only for the explicit E2E marker, so enabling the tool here cannot make
|
||||
# unrelated mock conversations pause.
|
||||
- name: 'e2e-hitl'
|
||||
label: 'E2E HITL'
|
||||
askUserQuestion: true
|
||||
preset:
|
||||
endpoint: 'Mock Provider A'
|
||||
model: 'mock-model-a'
|
||||
|
|
|
|||
9
e2e/playwright.config.bombadil.ts
Normal file
9
e2e/playwright.config.bombadil.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import { defineConfig } from '@playwright/test';
|
||||
import mockConfig from './playwright.config.mock';
|
||||
|
||||
export default defineConfig(mockConfig, {
|
||||
testDir: 'bombadil',
|
||||
testMatch: 'harness.spec.ts',
|
||||
retries: 0,
|
||||
reporter: [['list']],
|
||||
});
|
||||
|
|
@ -34,6 +34,7 @@ const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
|
|||
const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';
|
||||
const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:';
|
||||
const ACTIVITY_REPLY_MARKER = 'E2E_ACTIVITY_REPLY:';
|
||||
const ASK_USER_QUESTION_MARKER = 'E2E_ASK_USER_QUESTION:';
|
||||
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
|
||||
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
|
||||
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
|
||||
|
|
@ -68,6 +69,7 @@ const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';
|
|||
const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued';
|
||||
const ACTIVITY_FINAL_TEXT = 'E2E activity reply done';
|
||||
const STEER_TOOL_NAME_PREFIX = 'remember_fact';
|
||||
const ASK_USER_QUESTION_TOOL_NAME = 'ask_user_question';
|
||||
const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
|
||||
const ORDERED_CHUNK_DELAY_MS = 2;
|
||||
const ORDERED_REPLY_PIECES = 64;
|
||||
|
|
@ -1148,6 +1150,41 @@ function activityReplyResponses(label, toolNames) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause a real agent run at the ask_user_question tool. The resume controller
|
||||
* rebuilds the graph with an empty input-message list, so the test hook selects
|
||||
* its ordinary mock reply for the resumed model turn. This deliberately tests
|
||||
* the production checkpoint/resume seam rather than simulating a pause in the
|
||||
* browser fixture.
|
||||
*/
|
||||
function askUserQuestionResponses(label, toolNames) {
|
||||
if (!toolNames.has(ASK_USER_QUESTION_TOOL_NAME)) {
|
||||
return {
|
||||
responses: [
|
||||
`E2E ask user question unavailable: ${ASK_USER_QUESTION_TOOL_NAME} was not advertised.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: [''],
|
||||
toolCalls: [
|
||||
{
|
||||
id: `call_e2e_ask_user_question_${label}`,
|
||||
name: ASK_USER_QUESTION_TOOL_NAME,
|
||||
args: {
|
||||
question: `Which environment should Bombadil use for ${label}?`,
|
||||
description: 'This deterministic pause exercises the HITL answer and resume lifecycle.',
|
||||
options: [
|
||||
{ label: 'Staging', value: 'staging' },
|
||||
{ label: 'Production', value: 'production' },
|
||||
],
|
||||
},
|
||||
type: 'tool_call',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function findLastToolMessageText(messages, requiredToken) {
|
||||
for (let index = (messages ?? []).length - 1; index >= 0; index--) {
|
||||
const message = messages[index];
|
||||
|
|
@ -2023,6 +2060,11 @@ function resolveResponses({ graph, messages, text, toolNames }) {
|
|||
return activityReplyResponses(activityLabel, toolNames);
|
||||
}
|
||||
|
||||
const askUserQuestionLabel = getMarkerValue(text, ASK_USER_QUESTION_MARKER);
|
||||
if (askUserQuestionLabel) {
|
||||
return askUserQuestionResponses(askUserQuestionLabel, toolNames);
|
||||
}
|
||||
|
||||
if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {
|
||||
return {
|
||||
responses: [MOCK_REPLY],
|
||||
|
|
|
|||
11
package-lock.json
generated
11
package-lock.json
generated
|
|
@ -14,6 +14,7 @@
|
|||
"packages/*"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@antithesishq/bombadil": "0.6.1",
|
||||
"@axe-core/playwright": "^4.10.1",
|
||||
"@eslint/compat": "^1.2.6",
|
||||
"@eslint/eslintrc": "^3.3.4",
|
||||
|
|
@ -1255,6 +1256,16 @@
|
|||
"google-auth-library": "^9.4.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@antithesishq/bombadil": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",
|
||||
"integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"bombadil": "bin/bombadil.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@apideck/better-ajv-errors": {
|
||||
"version": "0.3.7",
|
||||
"resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.7.tgz",
|
||||
|
|
|
|||
13
package.json
13
package.json
|
|
@ -68,6 +68,18 @@
|
|||
"e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
|
||||
"e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts",
|
||||
"e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts",
|
||||
"e2e:bombadil": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:run": "playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:branch-reload": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:branch-reload:run": "cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:fork-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:fork-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:model-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:model-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:hitl": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:hitl:run": "cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:steering": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:bombadil:steering:run": "cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts",
|
||||
"e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts",
|
||||
"e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
|
||||
"e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts",
|
||||
|
|
@ -134,6 +146,7 @@
|
|||
},
|
||||
"homepage": "https://librechat.ai/",
|
||||
"devDependencies": {
|
||||
"@antithesishq/bombadil": "0.6.1",
|
||||
"@axe-core/playwright": "^4.10.1",
|
||||
"@eslint/compat": "^1.2.6",
|
||||
"@eslint/eslintrc": "^3.3.4",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue