diff --git a/.github/scripts/playwright-matrix.test.mjs b/.github/scripts/playwright-matrix.test.mjs new file mode 100644 index 0000000000..346741a6d8 --- /dev/null +++ b/.github/scripts/playwright-matrix.test.mjs @@ -0,0 +1,200 @@ +import yaml from 'js-yaml'; +import { test } from 'node:test'; +import { join } from 'node:path'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; + +const scriptsDir = fileURLToPath(new URL('.', import.meta.url)); +const workflow = yaml.load( + readFileSync(new URL('../workflows/playwright-mock.yml', import.meta.url), 'utf8'), +); +const selector = workflow.jobs.codegraph_select.steps.find((step) => step.id === 'sel'); +const shards = workflow.jobs.e2e_shards; +const defaults = [...shards.strategy.matrix.matchAll(/fromJSON\('([^']+)'\)/g)].map((match) => + JSON.parse(match[1]), +); +const prMatrix = JSON.parse(selector.env.FULL_INCLUDE); + +function assertPartition(matrix, store, suite, count) { + const lanes = matrix.include.filter( + (lane) => lane.stream_store === store && lane.suite === suite, + ); + assert.equal(lanes.length, count); + assert.deepEqual( + lanes.map((lane) => lane.shard), + Array.from({ length: count }, (_, index) => `${index + 1}/${count}`), + ); + assert.ok( + lanes.every((lane) => lane.redis_image === (store === 'redis' ? 'redis:7-alpine' : '')), + ); +} + +function select(t, response, overrides = {}) { + const scratch = mkdtempSync(join(scriptsDir, '.playwright-matrix-test-')); + t.after(() => rmSync(scratch, { recursive: true, force: true })); + writeFileSync(join(scratch, 'response.json'), JSON.stringify(response)); + writeFileSync(join(scratch, 'output'), ''); + writeFileSync(join(scratch, 'selector.sh'), selector.run); + writeFileSync( + join(scratch, 'gh'), + '#!/usr/bin/env bash\nprintf \'%s\\n\' \'{"path":"client/src/example.ts","status":"modified"}\'\nexit "${GH_EXIT:-0}"\n', + { mode: 0o700 }, + ); + writeFileSync( + join(scratch, 'curl'), + '#!/usr/bin/env bash\ncat response.json\nexit "${CURL_EXIT:-0}"\n', + { mode: 0o700 }, + ); + const result = spawnSync('bash', ['selector.sh'], { + cwd: scratch, + encoding: 'utf8', + timeout: 10000, + env: { + PATH: `${scratch}:${process.env.PATH}`, + FULL_INCLUDE: selector.env.FULL_INCLUDE, + URL: 'https://selector.invalid', + TOKEN: 'test-fixture', + REPO: 'fixture/repo', + PR: '1', + BASE_SHA: 'base', + HEAD_SHA: 'head', + CHANGED: '1', + E2E_SKIP_ARMED: '', + GITHUB_OUTPUT: join(scratch, 'output'), + GITHUB_STEP_SUMMARY: join(scratch, 'summary'), + ...overrides, + }, + }); + assert.equal(result.status, 0, result.stdout + result.stderr); + const outputs = Object.fromEntries( + readFileSync(join(scratch, 'output'), 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((line) => { + const separator = line.indexOf('='); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); + return { outputs, matrix: outputs.e2e_include ? JSON.parse(outputs.e2e_include) : defaults[0] }; +} + +function decision(redis, mcp = true) { + return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } }; +} + +test('PR selection and fail-open defaults use six memory shards and two Redis transport shards', () => { + assert.equal(defaults.length, 2); + assert.deepEqual(defaults[0], prMatrix); + assert.equal(prMatrix.include.length, 8); + assertPartition(prMatrix, 'memory', 'full', 6); + assertPartition(prMatrix, 'redis', 'transport', 2); +}); + +test('nightly and default manual runs retain full-suite coverage on both stores', () => { + assert.equal(workflow.on.workflow_dispatch.inputs.coverage.default, 'full'); + assert.equal(defaults[1].include.length, 4); + assertPartition(defaults[1], 'memory', 'full', 2); + assertPartition(defaults[1], 'redis', 'full', 2); +}); + +test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => { + assert.deepEqual(workflow.on.workflow_dispatch.inputs.coverage.options, ['full', 'pr']); + assert.match( + shards.strategy.matrix, + /github\.event_name == 'pull_request' \|\| inputs\.coverage == 'pr'/, + ); + assert.match(shards.if, /github\.event_name == 'workflow_dispatch'/); + assert.match(shards.if, /github\.event\.pull_request\.author_association/); +}); + +test('every shard has a unique job name and artifact namespace', () => { + for (const matrix of [prMatrix, ...defaults]) { + for (const key of ['name', 'artifact']) { + assert.equal(new Set(matrix.include.map((lane) => lane[key])).size, matrix.include.length); + } + } +}); + +test('both suite commands consume the matrix shard without increasing worker concurrency', () => { + for (const suite of ['full', 'transport']) { + const step = shards.steps.find((entry) => entry.if === `matrix.suite == '${suite}'`); + assert.match(step.run, /--shard=\$\{\{ matrix\.shard \}\}/); + assert.doesNotMatch(step.run, /--workers/); + } + const config = readFileSync( + new URL('../../e2e/playwright.config.mock.ts', import.meta.url), + 'utf8', + ); + assert.match(config, /workers: 1,/); + assert.equal(shards['runs-on'], 'ubuntu-latest'); +}); + +test('an explicit Redis skip drops both transport shards and no memory shard', (t) => { + const { matrix, outputs } = select(t, decision(false, false)); + assert.equal(outputs.decided, 'true'); + assert.equal(outputs.mcp_run, 'false'); + assert.equal(matrix.include.length, 6); + assertPartition(matrix, 'memory', 'full', 6); +}); + +test('missing or non-boolean Redis decisions retain both transport shards', (t) => { + for (const redis of [true, 'false', null, undefined]) { + const { matrix } = select(t, decision(redis)); + assert.deepEqual(matrix, prMatrix); + assertPartition(matrix, 'redis', 'transport', 2); + } +}); + +test('unavailable configuration, failed requests, and incomplete file lists fall back to the full PR matrix', (t) => { + for (const overrides of [ + { URL: '' }, + { TOKEN: '' }, + { CURL_EXIT: '28' }, + { GH_EXIT: '1' }, + { CHANGED: '2' }, + ]) { + const { matrix, outputs } = select(t, decision(false, false), overrides); + assert.equal(outputs.e2e_include, undefined); + assert.equal(outputs.mcp_run, undefined); + assert.deepEqual(matrix, prMatrix); + } + const { matrix, outputs } = select(t, {}); + assert.equal(outputs.e2e_include, undefined); + assert.deepEqual(matrix, prMatrix); +}); + +test('graduated spec skips keep the same pool across shards and remain opt-in', (t) => { + const response = { + ...decision(true), + e2e: { fail_open: false, graduated: ['e2e/specs/mock/sidebar.spec.ts'] }, + }; + const dark = select(t, response); + assert.equal(dark.outputs.e2e_skip, ''); + const armed = select(t, response, { E2E_SKIP_ARMED: 'on' }); + assert.equal(armed.outputs.e2e_skip, 'specs/mock/sidebar.spec.ts'); + assert.deepEqual(armed.matrix, prMatrix); + const fallback = select( + t, + { ...response, e2e: { ...response.e2e, fail_open: true } }, + { E2E_SKIP_ARMED: 'on' }, + ); + assert.equal(fallback.outputs.e2e_skip, ''); + assert.deepEqual(fallback.matrix, prMatrix); +}); + +test('the required gate still waits for all shards and rejects failures without fighting cancellation', () => { + const gate = workflow.jobs.e2e; + assert.equal(gate.name, 'e2e'); + assert.deepEqual(gate.needs, ['codegraph_select', 'e2e_shards', 'mcp_tool_list_changed']); + assert.equal(shards.strategy['fail-fast'], false); + assert.match(gate.if, /!cancelled\(\)/); + assert.match(shards.if, /!cancelled\(\)/); + assert.match(gate.steps[0].if, /needs\.e2e_shards\.result != 'success'/); + assert.match(gate.steps[0].if, /needs\.mcp_tool_list_changed\.result != 'success'/); + assert.match(gate.steps[0].if, /needs\.codegraph_select\.outputs\.mcp_run == 'false'/); + assert.equal(gate.steps[0].run, 'exit 1'); + assert.equal(workflow.concurrency['cancel-in-progress'], true); +}); diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index acb62732b1..9713220892 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -15,6 +15,13 @@ on: description: 'Reason for manual trigger' required: false default: 'Manual e2e run' + coverage: + description: 'Coverage profile: full suite on both stores, or the PR shards' + type: choice + options: + - full + - pr + default: full permissions: contents: read @@ -29,7 +36,7 @@ env: jobs: # Stage 2 of codegraph gating (stage 1 = backend jest in backend-review.yml). The graph decides - # two matrix lanes whose relevance is a reachability question: the redis-transport lane (the ten + # two matrix lanes whose relevance is a reachability question: the redis-transport shards (the # stream-boundary specs under a real Redis round-trip) and the MCP list_changed lanes. Memory # shards always run. Monotone and fail-open: a lane is dropped ONLY on an explicit `false` from # the service; unavailable/unconfigured/non-synchronize events keep the full matrix. Lock @@ -63,7 +70,7 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} CHANGED: ${{ github.event.pull_request.changed_files }} E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }} - FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}' + FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}' run: | set +e note() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; } @@ -112,7 +119,7 @@ jobs: INCLUDE="$FULL_INCLUDE" note "| redis transport | run |" fi - if ! echo "$INCLUDE" | jq -e '.include | length >= 3' >/dev/null 2>&1; then + if ! echo "$INCLUDE" | jq -e '.include | length >= 6' >/dev/null 2>&1; then note "_matrix assembly failed; running FULL_" exit 0 fi @@ -161,14 +168,16 @@ jobs: contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))) runs-on: ubuntu-latest timeout-minutes: 30 + # Separate runners isolate each shard's user, MongoDB, Redis and fixture state. + # Keep workers: 1 in Playwright; shared-user cleanup is not worker-safe. strategy: fail-fast: false matrix: >- ${{ (github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' && fromJSON(needs.codegraph_select.outputs.e2e_include)) || - (github.event_name == 'pull_request' && - fromJSON('{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}')) || + ((github.event_name == 'pull_request' || inputs.coverage == 'pr') && + fromJSON('{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) || fromJSON('{"include":[{"name":"memory, shard 1/2","stream_store":"memory","redis_image":"","suite":"full","shard":"1/2","artifact":"memory-1-of-2"},{"name":"memory, shard 2/2","stream_store":"memory","redis_image":"","suite":"full","shard":"2/2","artifact":"memory-2-of-2"},{"name":"redis, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/2","artifact":"redis-1-of-2"},{"name":"redis, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/2","artifact":"redis-2-of-2"}]}') }} services: @@ -211,6 +220,9 @@ jobs: if: steps.cache-node-modules.outputs.cache-hit != 'true' run: npm ci + - name: Test Playwright matrix selection + run: node --test .github/scripts/playwright-matrix.test.mjs + - name: Restore data-provider build cache id: cache-data-provider uses: actions/cache@v5 @@ -356,7 +368,7 @@ jobs: - name: Run Redis stream transport e2e if: matrix.suite == 'transport' - run: npx playwright test --config=e2e/playwright.config.redis.ts + run: npx playwright test --config=e2e/playwright.config.redis.ts --shard=${{ matrix.shard }} env: CI: 'true'