mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-09-01 19:41:32 +00:00
🎓 ci: Graduated E2E Spec Skipping, Wired Dark Until Armed (#15172)
Select emits e2e_skip from the decision's e2e.graduated (skippable-tier specs whose clean-trial streaks meet the pre-registered bar, 2x where history- coupled, >=3 distinct days — computed server-side from the shadow's per-spec ledger). Dark by default: the output is empty unless repo var CODEGRAPH_E2E_SKIP=on, and even armed it accepts only a well-typed pool-path list from a non-fail-open decision (traversal segments rejected). Shard steps subtract the skips from a git-derived run list — unknown names match nothing, skip-everything falls back to full, and skipped specs still execute post-merge in every full-suite vote run. 15 verbatim guard tests, scripts extracted from this YAML and executed against fixtures: arm/disarm, fail-open, malformed/missing/non-array lists, traversal, out-of-pool paths, unknown names, all-skips fallback, and the 2-real-skips 59->57 arg case. The traversal case caught a real regex gap pre-commit.
This commit is contained in:
parent
f9c051f8ea
commit
3df046f29a
1 changed files with 46 additions and 1 deletions
47
.github/workflows/playwright-mock.yml
vendored
47
.github/workflows/playwright-mock.yml
vendored
|
|
@ -49,6 +49,7 @@ jobs:
|
|||
decided: ${{ steps.sel.outputs.decided }}
|
||||
e2e_include: ${{ steps.sel.outputs.e2e_include }}
|
||||
mcp_run: ${{ steps.sel.outputs.mcp_run }}
|
||||
e2e_skip: ${{ steps.sel.outputs.e2e_skip }}
|
||||
steps:
|
||||
- name: Select matrix lanes, fail open on any doubt
|
||||
id: sel
|
||||
|
|
@ -61,6 +62,7 @@ jobs:
|
|||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
CHANGED: ${{ github.event.pull_request.changed_files }}
|
||||
E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }}
|
||||
FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}'
|
||||
run: |
|
||||
set +e
|
||||
|
|
@ -115,6 +117,25 @@ jobs:
|
|||
exit 0
|
||||
fi
|
||||
echo "e2e_include=$INCLUDE" >> "$GITHUB_OUTPUT"
|
||||
# Graduated per-spec skips are DARK until the operator arms repo variable
|
||||
# CODEGRAPH_E2E_SKIP=on (the election switch — flipped only when the pre-registered
|
||||
# resume condition holds). Even then, act only on a well-typed list from a non-fail-open
|
||||
# decision: every entry must be a pool spec path, or nothing is skipped. The server
|
||||
# already intersects with this PR's skippable tier and applies the streak bars
|
||||
# (2x where history-coupled); see codegraph-poc service/graduate.ts.
|
||||
SKIP=""
|
||||
if [ "$E2E_SKIP_ARMED" = "on" ]; then
|
||||
if echo "$RESP" | jq -e '(.e2e.fail_open != true) and (.e2e.graduated | type == "array" and all(.[]?; type == "string" and test("^e2e/specs/mock/[A-Za-z0-9._/-]+\\.spec\\.ts$") and (contains("..") | not)))' >/dev/null 2>&1; then
|
||||
SKIP=$(echo "$RESP" | jq -r '[.e2e.graduated[] | sub("^e2e/"; "")] | join(" ")')
|
||||
else
|
||||
note "_graduated list absent or malformed; no specs skipped_"
|
||||
fi
|
||||
fi
|
||||
echo "e2e_skip=$SKIP" >> "$GITHUB_OUTPUT"
|
||||
if [ -n "$SKIP" ]; then
|
||||
note "| graduated spec skips | $(echo "$SKIP" | wc -w | tr -d ' ') (armed) |"
|
||||
echo "codegraph-e2e-graduated-skips: $SKIP"
|
||||
fi
|
||||
echo "codegraph-select: redis_transport=$REDIS mcp_tool_list_changed=$MCP matrix_entries=$(echo "$INCLUDE" | jq '.include | length')"
|
||||
if [ "$MCP_SKIP" = 1 ]; then
|
||||
echo "mcp_run=false" >> "$GITHUB_OUTPUT"
|
||||
|
|
@ -305,9 +326,33 @@ jobs:
|
|||
|
||||
- name: Run full mock-LLM Tier-1 e2e
|
||||
if: matrix.suite == 'full'
|
||||
run: npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }}
|
||||
env:
|
||||
CI: 'true'
|
||||
E2E_SKIP: ${{ needs.codegraph_select.outputs.e2e_skip }}
|
||||
run: |
|
||||
set +e
|
||||
# Graduated-spec skipping (dark until repo var CODEGRAPH_E2E_SKIP=on upstream): subtract
|
||||
# the earned skips from a run list derived from the tree itself, so an unknown or stale
|
||||
# name in the skip list simply matches nothing. If subtraction would drop everything —
|
||||
# or drops nothing — run the full shard exactly as before. Skipped specs still execute
|
||||
# post-merge in every full-suite vote run, which is the net that catches a wrong skip.
|
||||
RUN_ARGS=""
|
||||
if [ -n "$E2E_SKIP" ]; then
|
||||
KEEP=""; DROP=0
|
||||
for spec in $(git ls-files 'e2e/specs/mock/*.spec.ts' 'e2e/specs/mock/**/*.spec.ts' | sed 's|^e2e/||' | sort -u); do
|
||||
case "$spec" in *" "*) KEEP="$KEEP $spec"; continue;; esac
|
||||
case " $E2E_SKIP " in
|
||||
*" $spec "*) DROP=$((DROP+1));;
|
||||
*) KEEP="$KEEP $spec";;
|
||||
esac
|
||||
done
|
||||
if [ "$DROP" -gt 0 ] && [ -n "$KEEP" ]; then
|
||||
RUN_ARGS="$KEEP"
|
||||
echo "codegraph-e2e-skip: dropped $DROP graduated specs from this shard's pool"
|
||||
fi
|
||||
fi
|
||||
set -e
|
||||
npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }} $RUN_ARGS
|
||||
|
||||
- name: Run Redis stream transport e2e
|
||||
if: matrix.suite == 'transport'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue