🎓 ci: Graduated E2E Spec Skipping, Wired Dark Until Armed (#15172)

Select emits e2e_skip from the decision's e2e.graduated (skippable-tier specs
whose clean-trial streaks meet the pre-registered bar, 2x where history-
coupled, >=3 distinct days — computed server-side from the shadow's per-spec
ledger). Dark by default: the output is empty unless repo var
CODEGRAPH_E2E_SKIP=on, and even armed it accepts only a well-typed pool-path
list from a non-fail-open decision (traversal segments rejected). Shard steps
subtract the skips from a git-derived run list — unknown names match nothing,
skip-everything falls back to full, and skipped specs still execute post-merge
in every full-suite vote run.

15 verbatim guard tests, scripts extracted from this YAML and executed against
fixtures: arm/disarm, fail-open, malformed/missing/non-array lists, traversal,
out-of-pool paths, unknown names, all-skips fallback, and the 2-real-skips
59->57 arg case. The traversal case caught a real regex gap pre-commit.
This commit is contained in:
Danny Avila 2026-08-24 11:53:50 -04:00 committed by GitHub
parent f9c051f8ea
commit 3df046f29a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -49,6 +49,7 @@ jobs:
decided: ${{ steps.sel.outputs.decided }}
e2e_include: ${{ steps.sel.outputs.e2e_include }}
mcp_run: ${{ steps.sel.outputs.mcp_run }}
e2e_skip: ${{ steps.sel.outputs.e2e_skip }}
steps:
- name: Select matrix lanes, fail open on any doubt
id: sel
@ -61,6 +62,7 @@ jobs:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
CHANGED: ${{ github.event.pull_request.changed_files }}
E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }}
FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}'
run: |
set +e
@ -115,6 +117,25 @@ jobs:
exit 0
fi
echo "e2e_include=$INCLUDE" >> "$GITHUB_OUTPUT"
# Graduated per-spec skips are DARK until the operator arms repo variable
# CODEGRAPH_E2E_SKIP=on (the election switch — flipped only when the pre-registered
# resume condition holds). Even then, act only on a well-typed list from a non-fail-open
# decision: every entry must be a pool spec path, or nothing is skipped. The server
# already intersects with this PR's skippable tier and applies the streak bars
# (2x where history-coupled); see codegraph-poc service/graduate.ts.
SKIP=""
if [ "$E2E_SKIP_ARMED" = "on" ]; then
if echo "$RESP" | jq -e '(.e2e.fail_open != true) and (.e2e.graduated | type == "array" and all(.[]?; type == "string" and test("^e2e/specs/mock/[A-Za-z0-9._/-]+\\.spec\\.ts$") and (contains("..") | not)))' >/dev/null 2>&1; then
SKIP=$(echo "$RESP" | jq -r '[.e2e.graduated[] | sub("^e2e/"; "")] | join(" ")')
else
note "_graduated list absent or malformed; no specs skipped_"
fi
fi
echo "e2e_skip=$SKIP" >> "$GITHUB_OUTPUT"
if [ -n "$SKIP" ]; then
note "| graduated spec skips | $(echo "$SKIP" | wc -w | tr -d ' ') (armed) |"
echo "codegraph-e2e-graduated-skips: $SKIP"
fi
echo "codegraph-select: redis_transport=$REDIS mcp_tool_list_changed=$MCP matrix_entries=$(echo "$INCLUDE" | jq '.include | length')"
if [ "$MCP_SKIP" = 1 ]; then
echo "mcp_run=false" >> "$GITHUB_OUTPUT"
@ -305,9 +326,33 @@ jobs:
- name: Run full mock-LLM Tier-1 e2e
if: matrix.suite == 'full'
run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }}
env:
CI: 'true'
E2E_SKIP: ${{ needs.codegraph_select.outputs.e2e_skip }}
run: |
set +e
# Graduated-spec skipping (dark until repo var CODEGRAPH_E2E_SKIP=on upstream): subtract
# the earned skips from a run list derived from the tree itself, so an unknown or stale
# name in the skip list simply matches nothing. If subtraction would drop everything —
# or drops nothing — run the full shard exactly as before. Skipped specs still execute
# post-merge in every full-suite vote run, which is the net that catches a wrong skip.
RUN_ARGS=""
if [ -n "$E2E_SKIP" ]; then
KEEP=""; DROP=0
for spec in $(git ls-files 'e2e/specs/mock/*.spec.ts' 'e2e/specs/mock/**/*.spec.ts' | sed 's|^e2e/||' | sort -u); do
case "$spec" in *" "*) KEEP="$KEEP $spec"; continue;; esac
case " $E2E_SKIP " in
*" $spec "*) DROP=$((DROP+1));;
*) KEEP="$KEEP $spec";;
esac
done
if [ "$DROP" -gt 0 ] && [ -n "$KEEP" ]; then
RUN_ARGS="$KEEP"
echo "codegraph-e2e-skip: dropped $DROP graduated specs from this shard's pool"
fi
fi
set -e
npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }} $RUN_ARGS
- name: Run Redis stream transport e2e
if: matrix.suite == 'transport'