🧑‍⚖️ ci: Require Explicit PR Quality Decisions and Validate Reports (#18025)

* ci: shard changed-function mutation tests
* ci: expand changed-function risk reports to backend packages
* ci: add PR commands and explicit quality decisions
* ci: queue concurrent quality decision updates
* ci: serialize quality requests and fix coverage setup
* ci: reuse Jest transforms across coverage groups
* ci: add check-all command and acknowledge quality requests
* ci: clarify quality scope and decision instructions
* ci: integrate quality commands with current dev checks
* fix: Keep Quality Decisions and Mutation Reports Fail Closed
This commit is contained in:
Atef Bellaaj 2026-11-12 11:42:15 +01:00 • committed by GitHub
parent 06c58c5fc1
commit edf0c416f3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 3558 additions and 1242 deletions

View file

@ -1,7 +1,34 @@
name: Change Quality
run-name: >-
${{ github.event_name == 'issue_comment' && format('Quality PR #{0} request {1}', github.event.issue.number, github.event.comment.id) || format('Change Quality {0}', github.ref_name) }}
on:
merge_group:
types: [checks_requested]
pull_request:
paths:
- 'scripts/quality/**'
- 'scripts/crap*'
- '.github/workflows/change-quality.yml'
- '.github/workflows/quality-decision.yml'
- 'package.json'
- 'package-lock.json'
- '.github/actions/mongodb-binary/**'
- 'packages/data-schemas/src/postgres/**'
- 'packages/data-schemas/src/types/favorite*'
- 'packages/data-schemas/src/methods/favorite*'
- 'packages/data-schemas/src/methods/index*'
- 'packages/data-schemas/jest.postgres*'
- 'packages/api/src/app/persistence*'
- 'packages/api/src/app/postgres*'
- 'packages/data-provider/src/config*'
- 'api/models/index*'
- 'api/server/index*'
- 'api/server/experimental*'
- 'config/postgres*'
- 'config/delete-user*'
issue_comment:
types: [created]
workflow_dispatch:
inputs:
base:
@ -9,15 +36,25 @@ on:
default: origin/dev
type: string
scope:
description: Test workspace (all or a workspace name)
default: data-schemas
description: Report scopes (all or comma-separated data-schemas,api,server)
default: all
type: string
check:
description: Checks to run
default: both
type: choice
options: [risk, mutation, both]
static:
description: Include static mutants
default: 'off'
type: choice
options: ['on', 'off']
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
group: ${{ github.workflow }}-${{ github.event_name == 'issue_comment' && github.run_id || github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
@ -25,7 +62,55 @@ env:
QUALITY_HEAD: ${{ github.event.pull_request.head.sha || github.sha }}
jobs:
queue-request:
name: Request queue quality decision
if: github.event_name == 'merge_group'
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- run: echo 'Quality Decision publishes this group from trusted default-branch code.'
request:
name: Accept quality request
concurrency:
group: quality-decision-${{ github.event.issue.number }}
cancel-in-progress: false
queue: max
if: >-
github.event_name == 'issue_comment' && github.event.issue.pull_request &&
startsWith(github.event.comment.body, '/quality') && github.event.comment.user.type != 'Bot'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
issues: write
pull-requests: write
actions: write
statuses: write
outputs:
accepted: ${{ steps.request.outputs.accepted }}
head: ${{ steps.request.outputs.head }}
base: ${{ steps.request.outputs.base }}
scope: ${{ steps.request.outputs.scope }}
check: ${{ steps.request.outputs.check }}
static: ${{ steps.request.outputs.static }}
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- id: request
env:
GH_TOKEN: ${{ github.token }}
run: node scripts/quality/comment.mts request
verify:
needs: [request]
if: always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true')
name: Verify quality tooling
runs-on: ubuntu-latest
timeout-minutes: 10
@ -48,11 +133,18 @@ jobs:
run: node --test scripts/crap.test.mts scripts/quality/*.test.mts
select:
needs: [request]
if: always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true')
env:
QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
name: Select changed source
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
workspaces: ${{ steps.select.outputs.workspaces }}
reports: ${{ steps.select.outputs.reports }}
mutationWorkspaces: ${{ steps.select.outputs.mutationWorkspaces }}
hasMutations: ${{ steps.select.outputs.hasMutations }}
hasChanges: ${{ steps.select.outputs.hasChanges }}
postgresChanged: ${{ steps.postgres.outputs.changed }}
steps:
@ -64,14 +156,21 @@ jobs:
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- name: Fetch requested commits
if: github.event_name == 'issue_comment'
run: |
git fetch --no-tags origin "$QUALITY_BASE" "$QUALITY_HEAD"
- name: Select workspaces
id: select
if: github.event_name != 'pull_request'
env:
QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }}
run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE"
QUALITY_SCOPE: ${{ needs.request.outputs.scope || inputs.scope || 'all' }}
QUALITY_CHECK: ${{ needs.request.outputs.check || inputs.check || 'both' }}
run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE" --check "${QUALITY_CHECK:-both}"
- name: Select PostgreSQL schema and lifecycle changes
id: postgres
if: github.event_name != 'issue_comment'
run: |
if git diff --name-only "$QUALITY_BASE...$QUALITY_HEAD" | grep -Eq '^(packages/data-schemas/src/(postgres/|types/favorite|methods/(favorite|index))|packages/api/src/app/(persistence|postgres)|packages/data-provider/src/config|api/(models/index|server/(index|experimental))|config/(postgres|delete-user)|packages/data-schemas/jest.postgres|package-lock.json|.github/workflows/change-quality.yml)'; then
echo 'changed=true' >> "$GITHUB_OUTPUT"
@ -80,7 +179,7 @@ jobs:
postgres:
name: PostgreSQL 16 contracts and readiness
needs: [select]
if: needs.select.outputs.postgresChanged == 'true'
if: always() && needs.select.result == 'success' && needs.select.outputs.postgresChanged == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
services:
@ -114,24 +213,19 @@ jobs:
- run: npx --no-install jest --config config/jest.config.js --runTestsByPath config/__tests__/delete-user.recovery.spec.js --coverage=false --runInBand
reports:
name: ${{ matrix.report.check == 'risk' && format('Change risk ({0})', matrix.workspace) || format('Mutation shard {0}/3 ({1})', matrix.report.shard, matrix.workspace) }}
needs: [select, verify]
if: needs.select.outputs.hasChanges == 'true'
name: ${{ matrix.task.check == 'risk' && 'Change risk' || format('Mutation testing {0}/4', matrix.task.shard) }} (${{ matrix.workspace }})
needs: [request, select, verify]
if: always() && needs.select.result == 'success' && needs.verify.result == 'success' && needs.select.outputs.hasChanges == 'true'
runs-on: ubuntu-latest
# Whole edited database functions can produce thousands of mutation cases.
timeout-minutes: ${{ matrix.report.check == 'mutation' && 240 || 45 }}
timeout-minutes: ${{ matrix.task.check == 'mutation' && 240 || 45 }}
defaults:
run:
working-directory: source
strategy:
fail-fast: false
matrix:
workspace: ${{ fromJSON(needs.select.outputs.workspaces) }}
report:
- { check: risk, shard: 0 }
- { check: mutation, shard: 1 }
- { check: mutation, shard: 2 }
- { check: mutation, shard: 3 }
include: ${{ fromJSON(needs.select.outputs.reports) }}
services:
redis:
image: redis:7-alpine
@ -152,6 +246,9 @@ jobs:
--health-timeout 5s
--health-retries 5
env:
QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
QUALITY_STATIC: ${{ needs.request.outputs.static || inputs.static || 'off' }}
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
REDIS_URI: redis://127.0.0.1:6379
NODE_OPTIONS: --max-old-space-size=6144
@ -168,6 +265,22 @@ jobs:
ref: ${{ github.sha }}
path: tooling
persist-credentials: false
- name: Start Redis cluster for API coverage
if: matrix.workspace == 'api' && matrix.task.check == 'risk'
working-directory: tooling/redis-config
timeout-minutes: 5
run: |
docker run --detach --network host --name quality-redis-cluster \
--volume "$PWD:/cluster" --workdir /cluster redis:7 \
bash -c 'bash start-cluster.sh && tail -f /dev/null'
for attempt in $(seq 1 30); do
if docker exec quality-redis-cluster redis-cli -p 7001 cluster info | grep -q 'cluster_state:ok'; then
exit 0
fi
sleep 1
done
docker logs quality-redis-cluster
exit 1
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
@ -181,15 +294,13 @@ jobs:
run: npm ci --ignore-scripts --workspaces=false
- name: Build shared packages
run: |
if [ "$QUALITY_SCOPE" = data-schemas ]; then
npm run build:data-provider
else
npm run build:packages
npm run build:data-provider
if [ "$QUALITY_WORKSPACE" != data-schemas ]; then
npm run build:data-schemas
npm run build:api
fi
env:
QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }}
# The action comes from the workflow revision like the rest of the tooling; the binary
# is the one the measured source's own install asks for.
QUALITY_WORKSPACE: ${{ matrix.workspace }}
- name: Provision MongoDB memory-server binary
uses: ./tooling/.github/actions/mongodb-binary
with:
@ -197,35 +308,43 @@ jobs:
- name: Generate advisory report
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
QUALITY_CHECK: ${{ matrix.report.check }}
QUALITY_SHARD: ${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}
QUALITY_CHECK: ${{ matrix.task.check }}
QUALITY_SHARD: ${{ matrix.task.shard }}
run: |
args=()
if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi
if [ "$QUALITY_CHECK" = mutation ]; then
args+=(--shard "$QUALITY_SHARD")
fi
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}"
# The mutation job downloads the mutation shards, so a shard that is not uploaded has to
# fail here. Nothing downloads the risk report, so losing it must not fail the job.
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" --static "${QUALITY_STATIC:-off}" "${args[@]}"
# Quality decisions require downloadable reports, so upload failures remain blocking.
- name: Upload reports
if: always()
continue-on-error: ${{ matrix.report.check == 'risk' }}
timeout-minutes: 3
uses: actions/upload-artifact@v6
with:
name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }}
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/
name: change-${{ matrix.task.check }}-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}${{ matrix.task.check == 'mutation' && format('-shard-{0}', matrix.task.shard) || '' }}
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.task.check }}${{ matrix.task.check == 'mutation' && format('-{0}', matrix.task.shard) || '' }}/
if-no-files-found: warn
retention-days: 14
overwrite: true
mutation:
name: Mutation testing (${{ matrix.workspace }})
needs: [select, reports]
if: always() && needs.select.result == 'success' && needs.select.outputs.hasChanges == 'true'
- name: Stop Redis cluster
if: always() && matrix.workspace == 'api' && matrix.task.check == 'risk'
run: docker rm -f quality-redis-cluster || true
mutation-summary:
name: Mutation report (${{ matrix.workspace }})
needs: [request, select, reports]
env:
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
if: always() && needs.select.outputs.hasMutations == 'true'
runs-on: ubuntu-latest
timeout-minutes: 5
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
workspace: ${{ fromJSON(needs.select.outputs.workspaces) }}
workspace: ${{ fromJSON(needs.select.outputs.mutationWorkspaces) }}
steps:
- uses: actions/checkout@v5
with:
@ -234,29 +353,54 @@ jobs:
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- uses: actions/download-artifact@v7
cache: npm
- run: npm ci
- uses: actions/download-artifact@v6
with:
pattern: change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}
path: mutation-reports
- name: Verify all mutation shards
pattern: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}-shard-*
path: coverage/shards
- name: Verify and combine mutation results
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
run: >-
node scripts/quality/aggregate.mts --directory mutation-reports --head "$QUALITY_HEAD"
--workspace "$QUALITY_WORKSPACE" --count 3
--output "coverage/changes/$QUALITY_WORKSPACE/mutation"
- name: Require successful report jobs
env:
REPORT_RESULT: ${{ needs.reports.result }}
run: test "$REPORT_RESULT" = success
# The aggregate is a debugging aid that no job downloads: see the note on the e2e_shards
# uploads in playwright-mock.yml.
node scripts/quality/merge.mts --directory coverage/shards
--workspace "$QUALITY_WORKSPACE"
--head "$QUALITY_HEAD"
--prefix "change-mutation-$QUALITY_WORKSPACE-$QUALITY_HEAD-"
- uses: actions/upload-artifact@v6
if: always()
timeout-minutes: 3
continue-on-error: true
with:
name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}
path: coverage/changes/${{ matrix.workspace }}/mutation/
if-no-files-found: warn
retention-days: 14
overwrite: true
progress:
name: Update quality progress
concurrency:
group: quality-decision-${{ github.event.issue.number }}
cancel-in-progress: false
queue: max
needs: [request, select, verify]
if: always() && needs.request.outputs.accepted == 'true'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
actions: write
pull-requests: write
statuses: write
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- env:
GH_TOKEN: ${{ github.token }}
run: node scripts/quality/comment.mts progress

87
.github/workflows/quality-decision.yml vendored Normal file
View file

@ -0,0 +1,87 @@
name: Quality Decision
on:
pull_request_target:
types: [opened, reopened, synchronize]
workflow_run:
workflows: [Change Quality]
types: [in_progress, completed]
permissions:
contents: read
actions: write
pull-requests: write
statuses: write
jobs:
resolve:
runs-on: ubuntu-latest
permissions: {}
outputs:
pr: ${{ steps.pr.outputs.pr }}
queue: ${{ steps.pr.outputs.queue }}
steps:
- id: pr
run: |
node <<'JS'
const fs = require('node:fs');
const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8'));
const run = event.workflow_run;
if (run?.event === 'merge_group' && run.status === 'completed') {
fs.appendFileSync(process.env.GITHUB_OUTPUT, `queue=${run.id}\n`);
}
const match = run?.event === 'issue_comment' && /^Quality PR #(\d+) request \d+$/.exec(run.display_title);
const pr = event.pull_request?.number ?? (match ? Number(match[1]) : undefined);
if (Number.isSafeInteger(pr) && pr > 0) fs.appendFileSync(process.env.GITHUB_OUTPUT, `pr=${pr}\n`);
JS
publish:
needs: [resolve]
if: needs.resolve.outputs.pr != ''
concurrency:
group: quality-decision-${{ needs.resolve.outputs.pr }}
cancel-in-progress: false
queue: max
name: Publish quality decision
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
# Never check out the PR or download its artifacts in this privileged job.
- uses: actions/checkout@v5
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- env:
GH_TOKEN: ${{ github.token }}
run: node scripts/quality/comment.mts publish
queue:
name: Publish queue quality decision
needs: [resolve]
if: needs.resolve.outputs.queue != ''
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: read
actions: read
statuses: write
concurrency:
group: quality-queue-${{ github.event.workflow_run.head_sha }}
cancel-in-progress: false
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.1'
- env:
GH_TOKEN: ${{ github.token }}
run: node scripts/quality/queue.mts