From fb8f71ab1497ff7cfca5b40c8a140932891e6fe8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 22 Oct 2026 11:28:55 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=B3=EF=B8=8F=20ci:=20Shard=20the=20Cod?= =?UTF-8?q?egraph=20E2E=20Vote=20Suite=20Across=20Ten=20Runners=20(#17339)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vote suite grew from 375 tests (39.1 min on one runner, 2026-09-30) to 1,321, so a single-runner vote can no longer finish. Run it as the same ten duration-planned shards playwright-mock.yml uses; each shard logs the specs it executed and the codegraph shadow unions them into one trial. A shard that runs out of its 28-minute test budget logs only its failed specs, never clean ones. --- .github/workflows/codegraph-e2e-votes.yml | 94 +++++++++++++++++++---- 1 file changed, 79 insertions(+), 15 deletions(-) diff --git a/.github/workflows/codegraph-e2e-votes.yml b/.github/workflows/codegraph-e2e-votes.yml index 0666338148..e46b18f372 100644 --- a/.github/workflows/codegraph-e2e-votes.yml +++ b/.github/workflows/codegraph-e2e-votes.yml @@ -20,6 +20,23 @@ # of 499 runs from 2026-09-24 never finished and graduation evidence stopped at 09-30. One # run proceeds and the newest merge waits as the single pending run. The whole campaign # switches off by setting repo variable CODEGRAPH_E2E_VOTES=off once the election passes. +# +# The suite runs as duration-planned shards, one runner each, the way playwright-mock.yml's +# memory shards do: on one runner with one worker it grew from 375 tests in 39.1 min +# (2026-09-30) to 1,321 tests, and run 39682542347 reached 570 of them when its 75-minute +# timeout cancelled it, so no vote finished after 09-30. Every shard plans the same disjoint +# partition (.github/scripts/playwright-shards.mjs over e2e/timings.mock.json), so the +# shards together still run the whole suite and the run is still ONE trial: the codegraph +# shadow unions the specs each shard's ledger logged. A shard that never logs its ledger +# (setup failed) adds no spec to the trial, so it can mint no clean trial for the specs it +# would have run. Each shard's test step has its own time budget below the job's, so a slow +# shard ends as a shard that ran out of time; a job timeout would cancel the whole run, and +# the shadow reads a cancelled run as no trial. A shard that ran out of time still logs the +# specs that FAILED: failing tests run three times, so a regression is what pushes a shard +# past its budget, and dropping that shard would drop exactly the failure the votes exist to +# catch. Its clean specs are not logged (not all of their tests ran), and the shadow ages +# each spec by its own newest observation, so a shard that never finishes stops its specs' +# skips instead of hiding behind the other shards' fresh votes. name: Codegraph E2E Votes on: @@ -45,15 +62,27 @@ env: jobs: vote: - name: vote (full suite) + name: vote (shard ${{ matrix.shard }}/10) if: vars.CODEGRAPH_E2E_VOTES != 'off' runs-on: ubuntu-latest - # The full suite took 35.8-45.0 min in the runs that finished; 45 cut 153 runs short. - timeout-minutes: 75 + # Sized from e2e/timings.mock.json: 6,151 recorded seconds over 10 shards plans ~10.3 min + # of tests per shard. PR CI's memory shards (same plan, same workers: 1) ran their test + # step in 8.3-16.5 min on 2026-10-21/22, up to 1.6x the plan, and the single vote job ran + # ~7.7 s per test (17 min for a tenth of 1,321). VOTE_TEST_MINUTES gives the tests 28 min + # (2.7x the plan); the job adds setup (about 2.5 min with npm ci, 70 s of it) and margin. + timeout-minutes: 40 + # Separate runners isolate each shard's user, MongoDB and fixture state. Keep workers: 1 + # in Playwright; shared-user cleanup is not worker-safe. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] env: CI: 'true' E2E_CHROMIUM_CHANNEL: chrome E2E_STREAM_STORE: memory + VOTE_SHARD: ${{ matrix.shard }}/10 + VOTE_TEST_MINUTES: '28' steps: - uses: actions/checkout@v5 @@ -180,15 +209,39 @@ jobs: continue-on-error: true run: .github/scripts/install-playwright-fonts.sh - - name: Vote — run the full mock suite (cannot fail the branch) + # The plan is playwright-mock.yml's: list the suite, assign whole spec files longest + # first to the least-loaded shard by recorded duration. No --skip (votes run every spec, + # graduated or not: they are the net that catches a wrong skip) and no --reserve (vote + # shards run nothing after the suite). + # + # The tests run under `timeout`, not the step's timeout-minutes, so the ledger can tell + # a shard that ran out of time from one whose tests failed: an interrupted shard's + # results would claim specs whose tests never all ran. + - name: Vote — run this shard of the mock suite (cannot fail the branch) continue-on-error: true - env: - # Absolute on purpose: Playwright resolves a relative PLAYWRIGHT_JSON_OUTPUT_NAME - # against the CONFIG directory (e2e/), not the working directory — the first live run - # wrote e2e/pw-results.json while the ledger looked in the repo root and logged zero - # trials (fail-safe, but a silent no-op). - PLAYWRIGHT_JSON_OUTPUT_NAME: ${{ github.workspace }}/pw-results.json - run: npx playwright test --config=e2e/playwright.config.mock.ts --reporter=line,json + run: | + PLAYWRIGHT_JSON_OUTPUT_FILE=e2e/.generated/playwright-list.json \ + npx playwright test --config=e2e/playwright.config.mock.ts --list --reporter=json > /dev/null + node .github/scripts/playwright-shards.mjs plan \ + --list e2e/.generated/playwright-list.json \ + --timings e2e/timings.mock.json \ + --shard "$VOTE_SHARD" \ + --out e2e/.generated/shard-tests.txt + # PLAYWRIGHT_JSON_OUTPUT_NAME is absolute on purpose: Playwright resolves a relative + # one against the CONFIG directory (e2e/), not the working directory — the first live + # run wrote e2e/pw-results.json while the ledger looked in the repo root and logged + # zero trials (fail-safe, but a silent no-op). + set +e + PLAYWRIGHT_JSON_OUTPUT_NAME="$GITHUB_WORKSPACE/pw-results.json" \ + timeout --signal=INT --kill-after=60 "${VOTE_TEST_MINUTES}m" \ + npx playwright test --config=e2e/playwright.config.mock.ts \ + --test-list=e2e/.generated/shard-tests.txt --reporter=line,json + rc=$? + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + touch "$GITHUB_WORKSPACE/vote-timed-out" + echo "::warning::shard $VOTE_SHARD ran out of its ${VOTE_TEST_MINUTES}-minute test budget" + fi + exit "$rc" - name: Ledger — log the specs that actually executed run: | @@ -199,15 +252,26 @@ jobs: # specs needs E2E_MODEL_SPECS_ENFORCE) are discovered by --list yet skip every test # under this job's default env — counting them as covered would mint phantom trials, # the exact bug this workflow revision exists to kill (Codex P1 on #15162). A spec is - # covered iff at least one of its tests reached a non-skipped outcome. - if jq -e '.suites' "$GITHUB_WORKSPACE/pw-results.json" >/dev/null 2>&1; then + # covered iff at least one of its tests reached a non-skipped outcome. Each shard logs + # only what it executed; the shadow unions the shards of one run into one trial. A + # shard that ran out of time logs no clean spec (not all of their tests ran), only + # the specs with a test Playwright counts as failed (outcome unexpected: every + # attempt that finished failed), so the shadow records them as failed trials. + if [ -e "$GITHUB_WORKSPACE/vote-timed-out" ]; then + echo "codegraph-votes: shard $VOTE_SHARD ran out of time — only its failures logged" + if jq -e '.suites' "$GITHUB_WORKSPACE/pw-results.json" >/dev/null 2>&1; then + jq -r '[.suites[] | recurse(.suites[]?) | .specs[]? | select(any(.tests[]?; .status == "unexpected")) | .file] | unique | .[]' "$GITHUB_WORKSPACE/pw-results.json" \ + | sed 's|^|specs/mock/|' > failed.txt + if [ -s failed.txt ]; then echo "codegraph-votes-failed: $(tr '\n' ' ' < failed.txt)"; fi + fi + elif jq -e '.suites' "$GITHUB_WORKSPACE/pw-results.json" >/dev/null 2>&1; then jq -r '[.suites[] | recurse(.suites[]?) | .specs[]? | select([.tests[]?.status] | any(. != "skipped")) | .file] | unique | .[]' "$GITHUB_WORKSPACE/pw-results.json" \ | sed 's|^|specs/mock/|' > covered.txt N=$(wc -l < covered.txt | tr -d ' ') - echo "codegraph-votes: running $N specs (executed, full suite)" + echo "codegraph-votes: running $N specs (executed, shard $VOTE_SHARD)" if [ "$N" != "0" ]; then echo "codegraph-votes-specs: $(tr '\n' ' ' < covered.txt)"; fi else - echo "codegraph-votes: no results json — run crashed before reporting; no trials logged" + echo "codegraph-votes: no results json — shard $VOTE_SHARD crashed before reporting; no trials logged" fi exit 0