From 3df046f29afc1e082422cb4dd58a2d8bf645f6b6 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 24 Aug 2026 11:53:50 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=93=20ci:=20Graduated=20E2E=20Spec=20S?= =?UTF-8?q?kipping,=20Wired=20Dark=20Until=20Armed=20(#15172)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select emits e2e_skip from the decision's e2e.graduated (skippable-tier specs whose clean-trial streaks meet the pre-registered bar, 2x where history- coupled, >=3 distinct days — computed server-side from the shadow's per-spec ledger). Dark by default: the output is empty unless repo var CODEGRAPH_E2E_SKIP=on, and even armed it accepts only a well-typed pool-path list from a non-fail-open decision (traversal segments rejected). Shard steps subtract the skips from a git-derived run list — unknown names match nothing, skip-everything falls back to full, and skipped specs still execute post-merge in every full-suite vote run. 15 verbatim guard tests, scripts extracted from this YAML and executed against fixtures: arm/disarm, fail-open, malformed/missing/non-array lists, traversal, out-of-pool paths, unknown names, all-skips fallback, and the 2-real-skips 59->57 arg case. The traversal case caught a real regex gap pre-commit. --- .github/workflows/playwright-mock.yml | 47 ++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 9e489d320a..25b6f1c8f7 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -49,6 +49,7 @@ jobs: decided: ${{ steps.sel.outputs.decided }} e2e_include: ${{ steps.sel.outputs.e2e_include }} mcp_run: ${{ steps.sel.outputs.mcp_run }} + e2e_skip: ${{ steps.sel.outputs.e2e_skip }} steps: - name: Select matrix lanes, fail open on any doubt id: sel @@ -61,6 +62,7 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} CHANGED: ${{ github.event.pull_request.changed_files }} + E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }} FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}' run: | set +e @@ -115,6 +117,25 @@ jobs: exit 0 fi echo "e2e_include=$INCLUDE" >> "$GITHUB_OUTPUT" + # Graduated per-spec skips are DARK until the operator arms repo variable + # CODEGRAPH_E2E_SKIP=on (the election switch — flipped only when the pre-registered + # resume condition holds). Even then, act only on a well-typed list from a non-fail-open + # decision: every entry must be a pool spec path, or nothing is skipped. The server + # already intersects with this PR's skippable tier and applies the streak bars + # (2x where history-coupled); see codegraph-poc service/graduate.ts. + SKIP="" + if [ "$E2E_SKIP_ARMED" = "on" ]; then + if echo "$RESP" | jq -e '(.e2e.fail_open != true) and (.e2e.graduated | type == "array" and all(.[]?; type == "string" and test("^e2e/specs/mock/[A-Za-z0-9._/-]+\\.spec\\.ts$") and (contains("..") | not)))' >/dev/null 2>&1; then + SKIP=$(echo "$RESP" | jq -r '[.e2e.graduated[] | sub("^e2e/"; "")] | join(" ")') + else + note "_graduated list absent or malformed; no specs skipped_" + fi + fi + echo "e2e_skip=$SKIP" >> "$GITHUB_OUTPUT" + if [ -n "$SKIP" ]; then + note "| graduated spec skips | $(echo "$SKIP" | wc -w | tr -d ' ') (armed) |" + echo "codegraph-e2e-graduated-skips: $SKIP" + fi echo "codegraph-select: redis_transport=$REDIS mcp_tool_list_changed=$MCP matrix_entries=$(echo "$INCLUDE" | jq '.include | length')" if [ "$MCP_SKIP" = 1 ]; then echo "mcp_run=false" >> "$GITHUB_OUTPUT" @@ -305,9 +326,33 @@ jobs: - name: Run full mock-LLM Tier-1 e2e if: matrix.suite == 'full' - run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }} env: CI: 'true' + E2E_SKIP: ${{ needs.codegraph_select.outputs.e2e_skip }} + run: | + set +e + # Graduated-spec skipping (dark until repo var CODEGRAPH_E2E_SKIP=on upstream): subtract + # the earned skips from a run list derived from the tree itself, so an unknown or stale + # name in the skip list simply matches nothing. If subtraction would drop everything — + # or drops nothing — run the full shard exactly as before. Skipped specs still execute + # post-merge in every full-suite vote run, which is the net that catches a wrong skip. + RUN_ARGS="" + if [ -n "$E2E_SKIP" ]; then + KEEP=""; DROP=0 + for spec in $(git ls-files 'e2e/specs/mock/*.spec.ts' 'e2e/specs/mock/**/*.spec.ts' | sed 's|^e2e/||' | sort -u); do + case "$spec" in *" "*) KEEP="$KEEP $spec"; continue;; esac + case " $E2E_SKIP " in + *" $spec "*) DROP=$((DROP+1));; + *) KEEP="$KEEP $spec";; + esac + done + if [ "$DROP" -gt 0 ] && [ -n "$KEEP" ]; then + RUN_ARGS="$KEEP" + echo "codegraph-e2e-skip: dropped $DROP graduated specs from this shard's pool" + fi + fi + set -e + npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }} $RUN_ARGS - name: Run Redis stream transport e2e if: matrix.suite == 'transport'