From edf0c416f33bfebec0a1560b1b947d2e90978747 Mon Sep 17 00:00:00 2001 From: Atef Bellaaj Date: Thu, 12 Nov 2026 11:42:15 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=91=E2=80=8D=E2=9A=96=EF=B8=8F=20ci:?= =?UTF-8?q?=20Require=20Explicit=20PR=20Quality=20Decisions=20and=20Valida?= =?UTF-8?q?te=20Reports=20(#18025)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: shard changed-function mutation tests * ci: expand changed-function risk reports to backend packages * ci: add PR commands and explicit quality decisions * ci: queue concurrent quality decision updates * ci: serialize quality requests and fix coverage setup * ci: reuse Jest transforms across coverage groups * ci: add check-all command and acknowledge quality requests * ci: clarify quality scope and decision instructions * ci: integrate quality commands with current dev checks * fix: Keep Quality Decisions and Mutation Reports Fail Closed --- .github/workflows/change-quality.yml | 250 +++++++-- .github/workflows/quality-decision.yml | 87 ++++ scripts/quality/README.md | 157 +++++- scripts/quality/aggregate.mts | 331 ------------ scripts/quality/aggregate.test.mts | 351 ------------- scripts/quality/changes.mts | 62 ++- scripts/quality/changes.test.mts | 92 +++- scripts/quality/comment.mts | 592 ++++++++++++++++++++++ scripts/quality/comment.test.mts | 676 +++++++++++++++++++++++++ scripts/quality/jest.mts | 84 ++- scripts/quality/merge.mts | 178 +++++++ scripts/quality/progress.mts | 11 +- scripts/quality/quality.test.mts | 135 ++++- scripts/quality/queue.mts | 230 +++++++++ scripts/quality/queue.test.mts | 284 +++++++++++ scripts/quality/run.mts | 161 +++--- scripts/quality/shard.mts | 30 -- scripts/quality/shard.test.mts | 234 --------- scripts/quality/shards.mts | 173 +++++++ scripts/quality/shards.test.mts | 243 +++++++++ scripts/quality/workflow.test.mts | 439 +++++++++++----- 21 files changed, 3558 insertions(+), 1242 deletions(-) create mode 100644 .github/workflows/quality-decision.yml delete mode 100644 scripts/quality/aggregate.mts delete mode 100644 scripts/quality/aggregate.test.mts create mode 100644 scripts/quality/comment.mts create mode 100644 scripts/quality/comment.test.mts create mode 100644 scripts/quality/merge.mts create mode 100644 scripts/quality/queue.mts create mode 100644 scripts/quality/queue.test.mts delete mode 100644 scripts/quality/shard.mts delete mode 100644 scripts/quality/shard.test.mts create mode 100644 scripts/quality/shards.mts create mode 100644 scripts/quality/shards.test.mts diff --git a/.github/workflows/change-quality.yml b/.github/workflows/change-quality.yml index e6dc959db9..0f568d1f8d 100644 --- a/.github/workflows/change-quality.yml +++ b/.github/workflows/change-quality.yml @@ -1,7 +1,34 @@ name: Change Quality +run-name: >- + ${{ github.event_name == 'issue_comment' && format('Quality PR #{0} request {1}', github.event.issue.number, github.event.comment.id) || format('Change Quality {0}', github.ref_name) }} on: + merge_group: + types: [checks_requested] pull_request: + paths: + - 'scripts/quality/**' + - 'scripts/crap*' + - '.github/workflows/change-quality.yml' + - '.github/workflows/quality-decision.yml' + - 'package.json' + - 'package-lock.json' + - '.github/actions/mongodb-binary/**' + - 'packages/data-schemas/src/postgres/**' + - 'packages/data-schemas/src/types/favorite*' + - 'packages/data-schemas/src/methods/favorite*' + - 'packages/data-schemas/src/methods/index*' + - 'packages/data-schemas/jest.postgres*' + - 'packages/api/src/app/persistence*' + - 'packages/api/src/app/postgres*' + - 'packages/data-provider/src/config*' + - 'api/models/index*' + - 'api/server/index*' + - 'api/server/experimental*' + - 'config/postgres*' + - 'config/delete-user*' + issue_comment: + types: [created] workflow_dispatch: inputs: base: @@ -9,15 +36,25 @@ on: default: origin/dev type: string scope: - description: Test workspace (all or a workspace name) - default: data-schemas + description: Report scopes (all or comma-separated data-schemas,api,server) + default: all type: string + check: + description: Checks to run + default: both + type: choice + options: [risk, mutation, both] + static: + description: Include static mutants + default: 'off' + type: choice + options: ['on', 'off'] permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.event_name == 'issue_comment' && github.run_id || github.event.pull_request.number || github.ref }} cancel-in-progress: true env: @@ -25,7 +62,55 @@ env: QUALITY_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} jobs: + queue-request: + name: Request queue quality decision + if: github.event_name == 'merge_group' + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - run: echo 'Quality Decision publishes this group from trusted default-branch code.' + + request: + name: Accept quality request + concurrency: + group: quality-decision-${{ github.event.issue.number }} + cancel-in-progress: false + queue: max + if: >- + github.event_name == 'issue_comment' && github.event.issue.pull_request && + startsWith(github.event.comment.body, '/quality') && github.event.comment.user.type != 'Bot' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + pull-requests: write + actions: write + statuses: write + outputs: + accepted: ${{ steps.request.outputs.accepted }} + head: ${{ steps.request.outputs.head }} + base: ${{ steps.request.outputs.base }} + scope: ${{ steps.request.outputs.scope }} + check: ${{ steps.request.outputs.check }} + static: ${{ steps.request.outputs.static }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: scripts/quality + - uses: actions/setup-node@v5 + with: + node-version: '24.22.3' + - id: request + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/quality/comment.mts request + verify: + needs: [request] + if: always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true') name: Verify quality tooling runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,11 +133,18 @@ jobs: run: node --test scripts/crap.test.mts scripts/quality/*.test.mts select: + needs: [request] + if: always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true') + env: + QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || inputs.base || 'origin/dev' }} + QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }} name: Select changed source runs-on: ubuntu-latest timeout-minutes: 5 outputs: - workspaces: ${{ steps.select.outputs.workspaces }} + reports: ${{ steps.select.outputs.reports }} + mutationWorkspaces: ${{ steps.select.outputs.mutationWorkspaces }} + hasMutations: ${{ steps.select.outputs.hasMutations }} hasChanges: ${{ steps.select.outputs.hasChanges }} postgresChanged: ${{ steps.postgres.outputs.changed }} steps: @@ -64,14 +156,21 @@ jobs: - uses: actions/setup-node@v5 with: node-version: '24.22.3' + - name: Fetch requested commits + if: github.event_name == 'issue_comment' + run: | + git fetch --no-tags origin "$QUALITY_BASE" "$QUALITY_HEAD" - name: Select workspaces id: select + if: github.event_name != 'pull_request' env: - QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }} - run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE" + QUALITY_SCOPE: ${{ needs.request.outputs.scope || inputs.scope || 'all' }} + QUALITY_CHECK: ${{ needs.request.outputs.check || inputs.check || 'both' }} + run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE" --check "${QUALITY_CHECK:-both}" - name: Select PostgreSQL schema and lifecycle changes id: postgres + if: github.event_name != 'issue_comment' run: | if git diff --name-only "$QUALITY_BASE...$QUALITY_HEAD" | grep -Eq '^(packages/data-schemas/src/(postgres/|types/favorite|methods/(favorite|index))|packages/api/src/app/(persistence|postgres)|packages/data-provider/src/config|api/(models/index|server/(index|experimental))|config/(postgres|delete-user)|packages/data-schemas/jest.postgres|package-lock.json|.github/workflows/change-quality.yml)'; then echo 'changed=true' >> "$GITHUB_OUTPUT" @@ -80,7 +179,7 @@ jobs: postgres: name: PostgreSQL 16 contracts and readiness needs: [select] - if: needs.select.outputs.postgresChanged == 'true' + if: always() && needs.select.result == 'success' && needs.select.outputs.postgresChanged == 'true' runs-on: ubuntu-latest timeout-minutes: 15 services: @@ -114,24 +213,19 @@ jobs: - run: npx --no-install jest --config config/jest.config.js --runTestsByPath config/__tests__/delete-user.recovery.spec.js --coverage=false --runInBand reports: - name: ${{ matrix.report.check == 'risk' && format('Change risk ({0})', matrix.workspace) || format('Mutation shard {0}/3 ({1})', matrix.report.shard, matrix.workspace) }} - needs: [select, verify] - if: needs.select.outputs.hasChanges == 'true' + name: ${{ matrix.task.check == 'risk' && 'Change risk' || format('Mutation testing {0}/4', matrix.task.shard) }} (${{ matrix.workspace }}) + needs: [request, select, verify] + if: always() && needs.select.result == 'success' && needs.verify.result == 'success' && needs.select.outputs.hasChanges == 'true' runs-on: ubuntu-latest # Whole edited database functions can produce thousands of mutation cases. - timeout-minutes: ${{ matrix.report.check == 'mutation' && 240 || 45 }} + timeout-minutes: ${{ matrix.task.check == 'mutation' && 240 || 45 }} defaults: run: working-directory: source strategy: fail-fast: false matrix: - workspace: ${{ fromJSON(needs.select.outputs.workspaces) }} - report: - - { check: risk, shard: 0 } - - { check: mutation, shard: 1 } - - { check: mutation, shard: 2 } - - { check: mutation, shard: 3 } + include: ${{ fromJSON(needs.select.outputs.reports) }} services: redis: image: redis:7-alpine @@ -152,6 +246,9 @@ jobs: --health-timeout 5s --health-retries 5 env: + QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || inputs.base || 'origin/dev' }} + QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }} + QUALITY_STATIC: ${{ needs.request.outputs.static || inputs.static || 'off' }} PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres REDIS_URI: redis://127.0.0.1:6379 NODE_OPTIONS: --max-old-space-size=6144 @@ -168,6 +265,22 @@ jobs: ref: ${{ github.sha }} path: tooling persist-credentials: false + - name: Start Redis cluster for API coverage + if: matrix.workspace == 'api' && matrix.task.check == 'risk' + working-directory: tooling/redis-config + timeout-minutes: 5 + run: | + docker run --detach --network host --name quality-redis-cluster \ + --volume "$PWD:/cluster" --workdir /cluster redis:7 \ + bash -c 'bash start-cluster.sh && tail -f /dev/null' + for attempt in $(seq 1 30); do + if docker exec quality-redis-cluster redis-cli -p 7001 cluster info | grep -q 'cluster_state:ok'; then + exit 0 + fi + sleep 1 + done + docker logs quality-redis-cluster + exit 1 - uses: actions/setup-node@v5 with: node-version: '24.22.3' @@ -181,15 +294,13 @@ jobs: run: npm ci --ignore-scripts --workspaces=false - name: Build shared packages run: | - if [ "$QUALITY_SCOPE" = data-schemas ]; then - npm run build:data-provider - else - npm run build:packages + npm run build:data-provider + if [ "$QUALITY_WORKSPACE" != data-schemas ]; then + npm run build:data-schemas + npm run build:api fi env: - QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }} - # The action comes from the workflow revision like the rest of the tooling; the binary - # is the one the measured source's own install asks for. + QUALITY_WORKSPACE: ${{ matrix.workspace }} - name: Provision MongoDB memory-server binary uses: ./tooling/.github/actions/mongodb-binary with: @@ -197,35 +308,43 @@ jobs: - name: Generate advisory report env: QUALITY_WORKSPACE: ${{ matrix.workspace }} - QUALITY_CHECK: ${{ matrix.report.check }} - QUALITY_SHARD: ${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }} + QUALITY_CHECK: ${{ matrix.task.check }} + QUALITY_SHARD: ${{ matrix.task.shard }} run: | args=() - if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi + if [ "$QUALITY_CHECK" = mutation ]; then + args+=(--shard "$QUALITY_SHARD") + fi node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \ - --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}" - # The mutation job downloads the mutation shards, so a shard that is not uploaded has to - # fail here. Nothing downloads the risk report, so losing it must not fail the job. + --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" --static "${QUALITY_STATIC:-off}" "${args[@]}" + # Quality decisions require downloadable reports, so upload failures remain blocking. - name: Upload reports if: always() - continue-on-error: ${{ matrix.report.check == 'risk' }} + timeout-minutes: 3 uses: actions/upload-artifact@v6 with: - name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }} - path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/ + name: change-${{ matrix.task.check }}-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}${{ matrix.task.check == 'mutation' && format('-shard-{0}', matrix.task.shard) || '' }} + path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.task.check }}${{ matrix.task.check == 'mutation' && format('-{0}', matrix.task.shard) || '' }}/ if-no-files-found: warn retention-days: 14 + overwrite: true - mutation: - name: Mutation testing (${{ matrix.workspace }}) - needs: [select, reports] - if: always() && needs.select.result == 'success' && needs.select.outputs.hasChanges == 'true' + - name: Stop Redis cluster + if: always() && matrix.workspace == 'api' && matrix.task.check == 'risk' + run: docker rm -f quality-redis-cluster || true + + mutation-summary: + name: Mutation report (${{ matrix.workspace }}) + needs: [request, select, reports] + env: + QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }} + if: always() && needs.select.outputs.hasMutations == 'true' runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: - workspace: ${{ fromJSON(needs.select.outputs.workspaces) }} + workspace: ${{ fromJSON(needs.select.outputs.mutationWorkspaces) }} steps: - uses: actions/checkout@v5 with: @@ -234,29 +353,54 @@ jobs: - uses: actions/setup-node@v5 with: node-version: '24.22.3' - - uses: actions/download-artifact@v7 + cache: npm + - run: npm ci + - uses: actions/download-artifact@v6 with: - pattern: change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }} - path: mutation-reports - - name: Verify all mutation shards + pattern: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}-shard-* + path: coverage/shards + - name: Verify and combine mutation results env: QUALITY_WORKSPACE: ${{ matrix.workspace }} run: >- - node scripts/quality/aggregate.mts --directory mutation-reports --head "$QUALITY_HEAD" - --workspace "$QUALITY_WORKSPACE" --count 3 - --output "coverage/changes/$QUALITY_WORKSPACE/mutation" - - name: Require successful report jobs - env: - REPORT_RESULT: ${{ needs.reports.result }} - run: test "$REPORT_RESULT" = success - # The aggregate is a debugging aid that no job downloads: see the note on the e2e_shards - # uploads in playwright-mock.yml. + node scripts/quality/merge.mts --directory coverage/shards + --workspace "$QUALITY_WORKSPACE" + --head "$QUALITY_HEAD" + --prefix "change-mutation-$QUALITY_WORKSPACE-$QUALITY_HEAD-" - uses: actions/upload-artifact@v6 if: always() timeout-minutes: 3 - continue-on-error: true with: name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }} path: coverage/changes/${{ matrix.workspace }}/mutation/ if-no-files-found: warn retention-days: 14 + overwrite: true + + progress: + name: Update quality progress + concurrency: + group: quality-decision-${{ github.event.issue.number }} + cancel-in-progress: false + queue: max + needs: [request, select, verify] + if: always() && needs.request.outputs.accepted == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: write + pull-requests: write + statuses: write + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: scripts/quality + - uses: actions/setup-node@v5 + with: + node-version: '24.22.3' + - env: + GH_TOKEN: ${{ github.token }} + run: node scripts/quality/comment.mts progress diff --git a/.github/workflows/quality-decision.yml b/.github/workflows/quality-decision.yml new file mode 100644 index 0000000000..bbb54a7aa8 --- /dev/null +++ b/.github/workflows/quality-decision.yml @@ -0,0 +1,87 @@ +name: Quality Decision + +on: + pull_request_target: + types: [opened, reopened, synchronize] + workflow_run: + workflows: [Change Quality] + types: [in_progress, completed] + +permissions: + contents: read + actions: write + pull-requests: write + statuses: write + +jobs: + resolve: + runs-on: ubuntu-latest + permissions: {} + outputs: + pr: ${{ steps.pr.outputs.pr }} + queue: ${{ steps.pr.outputs.queue }} + steps: + - id: pr + run: | + node <<'JS' + const fs = require('node:fs'); + const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')); + const run = event.workflow_run; + if (run?.event === 'merge_group' && run.status === 'completed') { + fs.appendFileSync(process.env.GITHUB_OUTPUT, `queue=${run.id}\n`); + } + const match = run?.event === 'issue_comment' && /^Quality PR #(\d+) request \d+$/.exec(run.display_title); + const pr = event.pull_request?.number ?? (match ? Number(match[1]) : undefined); + if (Number.isSafeInteger(pr) && pr > 0) fs.appendFileSync(process.env.GITHUB_OUTPUT, `pr=${pr}\n`); + JS + + publish: + needs: [resolve] + if: needs.resolve.outputs.pr != '' + concurrency: + group: quality-decision-${{ needs.resolve.outputs.pr }} + cancel-in-progress: false + queue: max + name: Publish quality decision + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # Never check out the PR or download its artifacts in this privileged job. + - uses: actions/checkout@v5 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + sparse-checkout: scripts/quality + - uses: actions/setup-node@v5 + with: + node-version: '24.22.3' + - env: + GH_TOKEN: ${{ github.token }} + run: node scripts/quality/comment.mts publish + + queue: + name: Publish queue quality decision + needs: [resolve] + if: needs.resolve.outputs.queue != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + actions: read + statuses: write + concurrency: + group: quality-queue-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: false + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + sparse-checkout: scripts/quality + - uses: actions/setup-node@v5 + with: + node-version: '24.22.1' + - env: + GH_TOKEN: ${{ github.token }} + run: node scripts/quality/queue.mts diff --git a/scripts/quality/README.md b/scripts/quality/README.md index 5d2c2256fc..ba0e163ec1 100644 --- a/scripts/quality/README.md +++ b/scripts/quality/README.md @@ -1,15 +1,39 @@ # Pull request quality reports -Every pull request runs two advisory checks for changed `packages/data-schemas` source. +PR commands and manual runs report change risk for added or edited functions in +`packages/data-schemas`, `packages/api`, and the backend `api` folder. +Mutation testing is experimental. It runs only for added or edited functions in +`packages/data-schemas`. Support for more scopes is planned. +Only workspaces with changed source get report jobs. PR updates do not start source +reports. PostgreSQL contract and readiness tests still run automatically for changes +to their schema and lifecycle paths. Report jobs also provide PostgreSQL and provision +the MongoDB test binary before tests start. Changes to the quality tooling still run its own verification tests automatically. Low scores do not fail a check. Failed tests, invalid reports, and tool errors do. -Each job has a 45-minute limit. A timeout is a failed run, not a quality score. +Mutation jobs have a 240-minute limit. Change-risk jobs have a 45-minute limit. +A workflow timeout is a failed run, not a quality score. A Stryker `Timeout` result +means a mutated program timed out; Stryker counts that mutation as detected. 1. Mutation testing uses Stryker 10.1.1. It mutates only added or edited functions. It selects related Jest tests and uses per-test coverage to reduce repeated work. - Static mutations remain enabled. These run outside an individual test, such as - during module loading or a shared test hook. Stryker reruns the related tests - for each mutated file. - The artifact includes an HTML report and a JSON report with mutation locations. + Static mutations are disabled by default. These run outside an individual test, + such as during module loading or a shared test hook. With `static=on`, Stryker + reruns the related tests for each mutated file. `static=off` sets Stryker's + `ignoreStatic: true`. Static-only mutants remain in the report as `Ignored`. + They do not affect the score. Mutants covered both during loading and inside + tests still run with per-test coverage. The exact-plan check includes ignored + results; it never treats a missing result as an ignored mutant. Each shard plan + records the static option. Shards with different options cannot be combined. + Four GitHub jobs run separate mutation ranges, with two Stryker workers per job. + The planner counts mutants before testing and assigns the largest ranges first + to the shard with the fewest mutants. Each original range stays intact: splitting + a block containing nested functions could remove its block-replacement mutation. + A large indivisible range can therefore make one shard slower than the others. + Sharding reduces waiting time but repeats setup and baseline tests on each runner. + It does not split or exclude the tests available to a mutation. + Each shard artifact includes an HTML report and a JSON report with mutation locations. + A final job combines the JSON reports and calculates one score from mutant counts. + It checks the head, plan, source, and every expected mutation. Missing shards, + duplicate mutations, runtime errors, or incomplete results prevent a combined score. The live Actions log shows progress every 30 seconds. It separates static mutations from other mutations and lists each unfinished function. A completion line shows the result counts for each finished function. Elapsed time and time since the last @@ -34,9 +58,12 @@ Each job has a 45-minute limit. A timeout is a failed run, not a quality score. It is a separate score and does not replace CC in the CRAP formula. Reports compare the PR head with its merge base. They do not measure the synthetic -GitHub merge commit. Both checkouts and commands use the same head SHA. The workflow -uses read-only permissions and needs no secrets. Reports appear in the Actions job -summary and in downloadable artifacts. It does not post PR comments. +GitHub merge commit. Application source and test dependencies come from the PR head. +Quality tooling and the combined-report job use the workflow revision, so older PR +heads do not need to contain the tooling scripts. Jobs that execute PR code have read-only permissions and need no secrets. Separate +jobs accept commands and publish comments and the merge status. They use trusted +workflow code, do not install PR dependencies, and do not open report artifacts. +Reports appear in the Actions job summary and in downloadable artifacts. The selector knows these source owners: `api`, `client`, `packages/api`, `packages/data-provider`, `packages/data-schemas`, `packages/client`, and `packages/chat`. Each uses its own Jest @@ -58,16 +85,114 @@ From the repository root, after `npm ci` and `npm run build:packages`: node scripts/quality/changes.mts --base origin/dev node scripts/quality/run.mts --base origin/dev --workspace data-schemas --check risk node scripts/quality/run.mts --base origin/dev --workspace data-schemas --check mutation +# Run one of the four CI shards (indices 1 through 4). +node scripts/quality/run.mts --base origin/dev --workspace data-schemas --check mutation --shard 1 node --test scripts/crap.test.mts scripts/quality/*.test.mts ``` -The automatic scope is `data-schemas`. -Other workspaces can be selected manually with `workflow_dispatch`; those mutation -configurations have not been validated by this trial. The selection summary lists the -workspaces excluded by the selected scope. `all` selects every changed application workspace. +The default command scope is `all`: change risk for `data-schemas`, `api` (`packages/api`), +and `server` (the backend `api` folder). A mixed change can run three risk jobs and +four data-schemas mutation shards. API-only or backend-only changes run no mutation jobs. +Manual `workflow_dispatch` runs can choose `risk`, `mutation`, or `both`, use `all` +or a comma-separated scope list, and set static mutants on or off. Mutation testing +stays limited to data-schemas in all cases. The selection summary +lists risk workspaces, mutation workspaces, and workspaces outside the scope. +Related Jest tests supply coverage for the changed files. Coverage uses each +workspace's existing worker count and memory recycling. API Redis integration +suites run separately with Redis enabled, using the same groups and serial lanes +as cache CI. The API report runner provides both the single Redis instance and the +repository's three-node Redis cluster. Istanbul merges the groups' coverage before +function scores are calculated. All groups use the same Jest coverage directory so +they can reuse compiled code. After each group, its raw reports move into its own +directory, including when tests fail. Every group still runs its tests with the +workspace's existing timeouts. Tests can exercise unchanged code, but only added +or edited functions appear in the risk report. + +## PR commands and merge decision + +A user with repository write access can post one of these commands as a new PR +conversation comment. Edited comments do not start or change a request. +The bot adds an eyes reaction to valid commands from authorized users. This includes +ignore and duplicate requests. If GitHub cannot add the reaction, processing continues. + +```text +/quality risk scopes=data-schemas,api,server +/quality mutation scopes=data-schemas static=off +/quality check-all +/quality check-all scopes=data-schemas static=on +/quality ignore +``` + +`scopes` defaults to `all`. `static` defaults to `off` and only affects mutation +runs. `all` means the supported scopes for the chosen check. Risk supports +`data-schemas`, `api` (`packages/api`), and `server` (the backend `api` folder). +Mutation supports only `data-schemas`. A `check-all` request runs both checks. It uses the selected scopes +for risk and includes mutation only when `data-schemas` is selected. +`/quality both` remains an alias for `/quality check-all`. The manual workflow and +local runner also default to static mutants off. +Unknown scopes, duplicate options, and invalid values are rejected before work starts. + +One accepted request can run per PR. Repeating the same effective command for the +same commit while it runs returns its existing link. Different valid options cancel +the previous request before new report jobs start. A finished request can be repeated. +Validation and the repository write-access check happen before cancellation. +Invalid commands and unauthorized users leave the current run and gate unchanged. +Cancellation only targets accepted comment requests from this repository and PR. +Other PRs, unrelated workflows, tooling verification, and standalone manual runs +are not cancelled by a request. + +The bot records the exact PR head and base commits when it accepts a request. +One bot comment shows the requested checks, effective scopes, static option, +requester, and Actions link. It updates after selection and tooling verification, +then again when the workflow ends. The final comment lists job outcomes and links +to report artifacts. Scores and function details stay in those reports and the +Actions summaries. Mutation logs provide the detailed 30-second progress updates. +A failed or cancelled workflow is shown as such. A new PR commit cancels old-commit work and marks earlier +request comments outdated. The current request can be rerun with its original +commit and options. Reruns of superseded requests or older commits are stopped. +Post a new command to measure the current commit or use different options. + +`Quality decision` is a commit status. It waits for an explicit decision on each +new PR head; it does not keep a runner idle. The latest accepted request for that +commit owns the status: + +4. A selected run clears the gate only after the workflow succeeds. A successful + empty selection also clears it. Low scores remain advisory. +5. Failed tests, tool errors, workflow timeouts, cancellations, or incomplete reports keep + the gate closed. Retry the run or make a new explicit decision. +6. `/quality ignore` cancels active quality work, then clears the gate and records + who chose to skip the checks. An older run finishing cannot replace it. +7. A new accepted run makes the gate wait again. A duplicate active command does + not change it. A new commit requires a new request or a new ignore decision. + Manual workflow dispatches do not count as explicit comment decisions. + +Request admission and status publication share one queued concurrency group per PR. Each reads the latest +head and decisions before it writes. This prevents an older result from reopening +the gate after a newer request has made it pending. + +## Activation + +The command and completion workflows must reach the repository default branch +(`main`) before GitHub can run their `issue_comment` and `workflow_run` triggers. +Merging this PR into `dev` alone does not activate those triggers. Follow the normal +`dev` to `main` promotion; do not backport the workflows separately. + +After live command tests pass, an administrator can add `Quality decision` as a +required status in the target branch's protection rule or ruleset. Select GitHub +Actions as its expected source. The status alone does not block merging until that +rule exists. The activation work remains part of PR #18025. Source reports now +require an explicit command or manual dispatch. Tooling verification remains +automatic for changes to the quality scripts, workflows, and dependencies. + +Live checks must cover a successful selected run, `static=off`, an invalid command, +a requester without write access, `/quality ignore`, cancellation, a failed test, +a newer commit during a run, and a rerun of an older commit. Confirm that comments +update in place and that only the current commit's latest decision clears the gate. +Also confirm that duplicate requests reuse their run and that replacements never +cancel another PR, unrelated CI, or standalone manual runs. Rule sources: -1. [Stryker configuration](https://stryker-mutator.io/docs/stryker-js/configuration/) -2. [Stryker Jest runner](https://stryker-mutator.io/docs/stryker-js/jest-runner/) -3. [SonarJS](https://github.com/SonarSource/SonarJS) +8. [Stryker configuration](https://stryker-mutator.io/docs/stryker-js/configuration/) +9. [Stryker Jest runner](https://stryker-mutator.io/docs/stryker-js/jest-runner/) +10. [SonarJS](https://github.com/SonarSource/SonarJS) diff --git a/scripts/quality/aggregate.mts b/scripts/quality/aggregate.mts deleted file mode 100644 index 932dbf1702..0000000000 --- a/scripts/quality/aggregate.mts +++ /dev/null @@ -1,331 +0,0 @@ -import { fileURLToPath } from 'node:url'; -import { join, resolve } from 'node:path'; -import { parseArgs, isDeepStrictEqual } from 'node:util'; -import { - appendFileSync, - mkdirSync, - readFileSync, - readdirSync, - rmSync, - writeFileSync, -} from 'node:fs'; - -import { shardTargets } from './shard.mts'; - -interface Options { - head: string; - workspace: string; - count: number; -} - -type ObjectValue = Record; -const statuses = [ - 'Killed', - 'Survived', - 'Timeout', - 'NoCoverage', - 'CompileError', - 'RuntimeError', - 'Ignored', - 'Pending', -] as const; -type Status = (typeof statuses)[number]; - -function requireValue(condition: unknown, message: string): asserts condition { - if (!condition) throw new Error(message); -} -function object(value: unknown): ObjectValue { - requireValue( - value !== null && typeof value === 'object' && !Array.isArray(value), - 'Expected JSON object', - ); - return value as ObjectValue; -} -function array(value: unknown): unknown[] { - requireValue(Array.isArray(value), 'Expected JSON array'); - return value; -} -function text(value: unknown): string { - requireValue(typeof value === 'string' && value.length > 0, 'Expected nonempty string'); - return value; -} -function integer(value: unknown, minimum = 0): number { - requireValue( - Number.isSafeInteger(value) && typeof value === 'number' && value >= minimum, - 'Expected nonnegative integer', - ); - return value; -} -function path(value: unknown): string { - const file = text(value); - requireValue( - !file.includes('\\') && - !file.startsWith('/') && - file.split('/').every((part) => part && part !== '.' && part !== '..'), - 'Expected repository-relative file', - ); - return file; -} -function location(value: unknown): ObjectValue { - const span = object(value); - const start = object(span.start); - const end = object(span.end); - const line = integer(start.line, 1); - const column = integer(start.column, 1); - const endLine = integer(end.line, 1); - const endColumn = integer(end.column, 1); - requireValue(endLine > line || (endLine === line && endColumn >= column), 'Invalid location'); - return span; -} -function targets(value: unknown) { - const files = new Set(); - return array(value).map((entry) => { - const target = object(entry); - const file = path(target.file); - requireValue(!files.has(file), 'Duplicate target file'); - files.add(file); - const functions = array(target.functions); - requireValue(functions.length > 0 && array(target.mutate).length > 0, 'Empty mutation target'); - for (const fn of functions) { - const row = object(fn); - text(row.function); - integer(row.line, 1); - integer(row.column, 1); - location(row.location); - } - for (const pattern of array(target.mutate)) text(pattern); - return { file, functions, original: target }; - }); -} - -/** Validate complete file shards without changing their reports or local test/mutant IDs. */ -export function aggregate(inputs: readonly unknown[], options: Options) { - integer(options.count, 1); - text(options.head); - text(options.workspace); - requireValue(inputs.length === options.count, 'Wrong shard directory count'); - const shards = inputs - .map((input) => { - const row = object(input); - const selection = object(row.selection); - const shard = object(selection.shard); - const index = integer(shard.index, 1); - requireValue( - index <= options.count && shard.total === options.count, - 'Wrong shard index or count', - ); - requireValue(selection.head === options.head, 'Wrong selection head'); - requireValue(selection.workspace === options.workspace, 'Wrong selection workspace'); - text(selection.base); - const files = array(selection.files).map((entry) => path(object(entry).file)); - requireValue(new Set(files).size === files.length, 'Duplicate selected file'); - const all = targets(row.allFunctions); - requireValue( - all.every(({ file }) => files.includes(file)), - 'Target outside selected files', - ); - targets(row.functions); - const expected = shardTargets(all, { index, total: options.count }); - requireValue( - isDeepStrictEqual( - row.functions, - expected.map(({ original }) => original), - ), - 'Wrong file/function shard partition', - ); - return { - index, - selection, - all, - expected, - report: object(row.report), - progress: object(row.progress), - }; - }) - .sort((a, b) => a.index - b.index); - requireValue( - new Set(shards.map(({ index }) => index)).size === options.count, - 'Duplicate shard index', - ); - const first = shards[0]!; - const counts: Record = { - Killed: 0, - Survived: 0, - Timeout: 0, - NoCoverage: 0, - CompileError: 0, - RuntimeError: 0, - Ignored: 0, - Pending: 0, - }; - const identities = new Set(); - let staticTotal = 0; - for (const shard of shards) { - requireValue(shard.selection.base === first.selection.base, 'Inconsistent selection base'); - requireValue( - isDeepStrictEqual(shard.selection.files, first.selection.files), - 'Inconsistent original selected files', - ); - requireValue( - isDeepStrictEqual( - shard.all.map(({ original }) => original), - first.all.map(({ original }) => original), - ), - 'Inconsistent all-functions manifest', - ); - requireValue(shard.report.schemaVersion === '1.0', 'Unsupported mutation report schema'); - let total = 0; - let staticCount = 0; - const ids = new Set(); - for (const [file, entry] of Object.entries(object(shard.report.files))) { - path(file); - requireValue( - shard.expected.some((target) => target.file === file), - 'Report file outside shard targets', - ); - for (const value of array(object(entry).mutants)) { - const mutant = object(value); - const id = text(mutant.id); - requireValue(!ids.has(id), 'Duplicate local mutant ID'); - ids.add(id); - const span = location(mutant.location); - const mutator = text(mutant.mutatorName); - requireValue(typeof mutant.replacement === 'string', 'Invalid mutant replacement'); - requireValue( - mutant.static === undefined || typeof mutant.static === 'boolean', - 'Invalid static flag', - ); - const status = statuses.find((name) => name === mutant.status); - requireValue(status, 'Unknown mutant status'); - requireValue( - status !== 'RuntimeError' && status !== 'Pending', - 'Incomplete mutant outcome', - ); - const identity = JSON.stringify([ - file, - object(span.start).line, - object(span.start).column, - object(span.end).line, - object(span.end).column, - mutator, - mutant.replacement, - ]); - requireValue(!identities.has(identity), 'Duplicate mutation identity'); - identities.add(identity); - counts[status]++; - total++; - if (mutant.static === true) staticCount++; - } - } - const progress = shard.progress; - requireValue( - progress.complete === true && progress.completed === total && progress.total === total, - 'Incomplete mutation progress', - ); - requireValue( - progress.staticCompleted === staticCount && progress.staticTotal === staticCount, - 'Incomplete static mutation progress', - ); - staticTotal += staticCount; - } - const detected = counts.Killed + counts.Timeout; - const tested = detected + counts.Survived + counts.NoCoverage; - const total = Object.values(counts).reduce((sum, count) => sum + count, 0); - return { - head: options.head, - base: first.selection.base, - workspace: options.workspace, - shardCount: options.count, - complete: true, - files: first.selection.files, - functions: first.all.map(({ original }) => original), - total, - completed: total, - staticTotal, - staticCompleted: staticTotal, - counts, - score: tested ? (100 * detected) / tested : null, - shards: shards.map(({ index, report, progress }) => ({ index, report, progress })), - }; -} - -export function aggregateMarkdown(result: ReturnType): string { - const score = result.score === null ? 'N/A (no valid mutants)' : `${result.score.toFixed(2)}%`; - return [ - `## ${result.workspace}: mutation testing (advisory)`, - '', - `Complete: ${result.shardCount} shards, ${result.completed}/${result.total} mutants. Static mutations: ${result.staticCompleted}/${result.staticTotal}.`, - '', - `Mutation score: ${score}. Compile errors and ignored mutants are excluded from the score, not the report.`, - '', - '| Result | Mutants |', - '| --- | ---: |', - ...statuses.map((status) => `| ${status} | ${result.counts[status]} |`), - '', - 'Aggregate JSON retains each shard report, including source, test metadata, and unchanged local IDs.', - '', - ].join('\n'); -} - -function main() { - const { values } = parseArgs({ - options: { - directory: { type: 'string' }, - head: { type: 'string' }, - workspace: { type: 'string' }, - count: { type: 'string' }, - output: { type: 'string' }, - }, - }); - requireValue( - values.directory && values.head && values.workspace && values.count && values.output, - 'Required: --directory --head --workspace --count --output', - ); - for (const name of ['aggregate.json', 'summary.md']) { - rmSync(join(values.output, name), { force: true }); - } - requireValue(/^[1-9]\d*$/u.test(values.count), 'Invalid shard count'); - const entries = readdirSync(values.directory, { withFileTypes: true }); - requireValue( - entries.length === Number(values.count) && entries.every((entry) => entry.isDirectory()), - 'Wrong shard directory count', - ); - const inputs = entries.map((entry) => { - const read = (name: string): unknown => { - try { - return JSON.parse(readFileSync(join(values.directory!, entry.name, name), 'utf8')); - } catch { - throw new Error(`Missing or malformed shard ${name}`); - } - }; - return { - selection: read('selection.json'), - allFunctions: read('all-functions.json'), - functions: read('functions.json'), - report: read('mutation.json'), - progress: read('progress.json'), - }; - }); - const result = aggregate(inputs, { - head: values.head, - workspace: values.workspace, - count: Number(values.count), - }); - mkdirSync(values.output, { recursive: true }); - writeFileSync(join(values.output, 'aggregate.json'), JSON.stringify(result, null, 2)); - const markdown = aggregateMarkdown(result); - writeFileSync(join(values.output, 'summary.md'), markdown); - if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); - console.log(markdown); -} - -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - try { - main(); - } catch { - console.error( - 'Mutation aggregation failed: shard evidence is missing, inconsistent, malformed, or incomplete.', - ); - process.exitCode = 1; - } -} diff --git a/scripts/quality/aggregate.test.mts b/scripts/quality/aggregate.test.mts deleted file mode 100644 index 1f7369ac31..0000000000 --- a/scripts/quality/aggregate.test.mts +++ /dev/null @@ -1,351 +0,0 @@ -import { test } from 'node:test'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { strict as assert } from 'node:assert'; -import { spawnSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; - -import { aggregate, aggregateMarkdown } from './aggregate.mts'; -import { shardTargets } from './shard.mts'; - -const options = { head: 'head-sha', workspace: 'api', count: 3 }; -const point = (line: number) => ({ line, column: 1 }); -const span = (line: number) => ({ start: point(line), end: point(line + 1) }); -const functions = Array.from({ length: 5 }, (_, index) => ({ - file: `packages/api/src/file${index}.ts`, - functions: Array.from({ length: [2, 4, 1, 3, 2][index] }, (_, fn) => fn * 2 + 1).map((line) => ({ - function: `f${line}`, - line, - column: 1, - location: span(line), - })), - mutate: [`packages/api/src/file${index}.ts:1:0-4:0`], -})); -const files = functions.map(({ file }) => ({ file, ranges: [{ start: 1, count: 4 }] })); -const outcomes = ['Killed', 'Survived', 'Timeout', 'NoCoverage', 'CompileError', 'Ignored']; -function fixtures() { - return Array.from({ length: 3 }, (_, shard) => { - const chosen = shardTargets(functions, { index: shard + 1, total: 3 }); - const reportFiles = Object.fromEntries( - chosen.map(({ file }, index) => [ - file, - { - source: 'function source() {}', - language: 'typescript', - mutants: [0, 1].map((position) => ({ - id: String(index * 2 + position), - location: span(position + 1), - mutatorName: 'BooleanLiteral', - replacement: 'false', - status: outcomes[(shard * 2 + position) % outcomes.length], - static: position === 1, - coveredBy: ['0'], - killedBy: position === 0 ? ['0'] : [], - statusReason: 'preserve reason', - })), - }, - ]), - ); - const total = chosen.length * 2; - return { - selection: { - head: options.head, - base: 'base-sha', - workspace: options.workspace, - files: structuredClone(files), - shard: { index: shard + 1, total: 3 }, - }, - allFunctions: structuredClone(functions), - functions: structuredClone([...chosen]), - report: { - schemaVersion: '1.0', - files: reportFiles, - testFiles: { 'tests/file.ts': { tests: [{ id: '0', name: 'test' }] } }, - framework: { name: 'Stryker' }, - }, - progress: { - complete: true, - completed: total, - total, - staticCompleted: chosen.length, - staticTotal: chosen.length, - }, - }; - }); -} - -test('aggregates every file, function, outcome, static mutant, and unchanged raw report', () => { - const inputs = fixtures(); - const snapshot = structuredClone(inputs); - const result = aggregate(inputs, options); - assert.equal(result.complete, true); - assert.equal(result.total, 10); - assert.equal(result.completed, 10); - assert.equal(result.staticTotal, 5); - assert.equal(result.staticCompleted, 5); - assert.deepEqual(result.functions, functions); - assert.deepEqual(result.files, files); - assert.deepEqual(result.counts, { - Killed: 2, - Survived: 2, - Timeout: 2, - NoCoverage: 2, - CompileError: 1, - RuntimeError: 0, - Ignored: 1, - Pending: 0, - }); - assert.equal(result.score, 50); - assert.deepEqual( - result.shards.map(({ report }) => report), - inputs.map(({ report }) => report), - ); - assert.deepEqual(inputs, snapshot); - assert.match(aggregateMarkdown(result), /CompileError \| 1/); - assert.match(aggregateMarkdown(result), /Static mutations: 5\/5/); -}); - -test('shard directory ordering does not affect results or identities', () => { - const inputs = fixtures(); - assert.deepEqual( - aggregate([inputs[2], inputs[0], inputs[1]], options), - aggregate(inputs, options), - ); -}); - -const invalid: [string, (inputs: ReturnType) => void][] = [ - [ - 'missing shard', - (inputs) => { - inputs.pop(); - }, - ], - [ - 'extra shard', - (inputs) => { - inputs.push(structuredClone(inputs[0]!)); - }, - ], - [ - 'duplicate shard index', - (inputs) => { - inputs[1]!.selection.shard.index = 1; - }, - ], - [ - 'wrong shard count', - (inputs) => { - inputs[0]!.selection.shard.total = 4; - }, - ], - [ - 'wrong head', - (inputs) => { - inputs[0]!.selection.head = 'other'; - }, - ], - [ - 'inconsistent base', - (inputs) => { - inputs[1]!.selection.base = 'other'; - }, - ], - [ - 'wrong workspace', - (inputs) => { - inputs[0]!.selection.workspace = 'client'; - }, - ], - [ - 'inconsistent original files', - (inputs) => { - inputs[0]!.selection.files.pop(); - }, - ], - [ - 'inconsistent full manifest', - (inputs) => { - inputs[1]!.allFunctions[0]!.functions[0]!.function = 'other'; - }, - ], - [ - 'dropped target file', - (inputs) => { - inputs[0]!.functions.pop(); - }, - ], - [ - 'dropped target function', - (inputs) => { - inputs[0]!.functions[0]!.functions.pop(); - }, - ], - [ - 'wrong round-robin partition', - (inputs) => { - inputs[0]!.functions.reverse(); - }, - ], - [ - 'out-of-shard report file', - (inputs) => { - inputs[0]!.report.files['packages/api/src/other.ts'] = structuredClone( - Object.values(inputs[0]!.report.files)[0]!, - ); - }, - ], - [ - 'runtime error', - (inputs) => { - Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'RuntimeError'; - }, - ], - [ - 'pending mutant', - (inputs) => { - Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Pending'; - }, - ], - [ - 'unknown status', - (inputs) => { - Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Unknown'; - }, - ], - [ - 'incomplete progress', - (inputs) => { - inputs[0]!.progress.complete = false; - }, - ], - [ - 'wrong completed count', - (inputs) => { - inputs[0]!.progress.completed--; - }, - ], - [ - 'wrong planned count', - (inputs) => { - inputs[0]!.progress.total++; - }, - ], - [ - 'wrong static completed count', - (inputs) => { - inputs[0]!.progress.staticCompleted--; - }, - ], - [ - 'wrong static total count', - (inputs) => { - inputs[0]!.progress.staticTotal++; - }, - ], - [ - 'duplicate mutant identity', - (inputs) => { - const mutants = Object.values(inputs[0]!.report.files)[0]!.mutants; - mutants[1] = { ...mutants[0]!, id: 'new' }; - }, - ], - [ - 'duplicate local ID', - (inputs) => { - Object.values(inputs[0]!.report.files)[0]!.mutants[1]!.id = '0'; - }, - ], -]; -for (const [name, change] of invalid) { - test(`fails closed: ${name}`, () => { - const inputs = fixtures(); - change(inputs); - assert.throws(() => aggregate(inputs, options)); - }); -} - -test('requires explicit complete evidence for zero-mutant shards', () => { - const inputs = fixtures(); - for (const input of inputs) { - input.allFunctions = []; - input.functions = []; - input.report.files = {}; - input.progress = { complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 }; - } - const result = aggregate(inputs, options); - assert.equal(result.total, 0); - assert.equal(result.score, null); - assert.deepEqual(result.functions, []); - const missing: unknown[] = structuredClone(inputs); - missing[0] = { ...inputs[0], progress: undefined }; - assert.throws(() => aggregate(missing, options)); -}); - -test('chosen function files may legitimately produce no mutants', () => { - const inputs = fixtures(); - inputs[0]!.report.files = {}; - inputs[0]!.progress = { - complete: true, - completed: 0, - total: 0, - staticCompleted: 0, - staticTotal: 0, - }; - assert.equal(aggregate(inputs, options).total, 6); -}); - -test('malformed evidence is rejected without trusting TypeScript casts', () => { - for (const bad of [null, [], {}, { selection: null }]) { - assert.throws(() => aggregate([bad, bad, bad], options)); - } - const inputs = fixtures(); - assert.throws(() => aggregate(inputs, { ...options, count: 0 })); - assert.throws(() => aggregate(inputs, { ...options, head: '' })); -}); - -test('CLI uses only built-ins, reads unmerged directories, and fails without a green output', (t) => { - const directory = mkdtempSync(join(tmpdir(), 'mutation-aggregate-')); - t.after(() => rmSync(directory, { recursive: true, force: true })); - const input = join(directory, 'reports'); - const output = join(directory, 'output'); - for (const [index, shard] of fixtures().entries()) { - const folder = join(input, `artifact-${2 - index}`); - mkdirSync(folder, { recursive: true }); - for (const [name, value] of Object.entries({ - selection: shard.selection, - 'all-functions': shard.allFunctions, - functions: shard.functions, - mutation: shard.report, - progress: shard.progress, - })) { - writeFileSync(join(folder, `${name}.json`), JSON.stringify(value)); - } - } - const args = [ - 'scripts/quality/aggregate.mts', - '--directory', - input, - '--head', - options.head, - '--workspace', - options.workspace, - '--count', - '3', - '--output', - output, - ]; - const stepSummary = join(directory, 'step-summary.md'); - const result = spawnSync(process.execPath, args, { - encoding: 'utf8', - env: { ...process.env, GITHUB_STEP_SUMMARY: stepSummary }, - }); - assert.equal(result.status, 0, result.stderr); - assert.equal(JSON.parse(readFileSync(join(output, 'aggregate.json'), 'utf8')).total, 10); - assert.match(readFileSync(join(output, 'summary.md'), 'utf8'), /50.00%/); - assert.equal(readFileSync(stepSummary, 'utf8'), readFileSync(join(output, 'summary.md'), 'utf8')); - writeFileSync(join(input, 'artifact-0', 'progress.json'), 'not-json-secret-payload'); - const failed = spawnSync(process.execPath, args, { encoding: 'utf8' }); - assert.equal(failed.status, 1); - assert.doesNotMatch(failed.stderr, /secret-payload/); - assert.throws(() => readFileSync(join(output, 'summary.md'))); -}); diff --git a/scripts/quality/changes.mts b/scripts/quality/changes.mts index 92e3aa4777..60775605a3 100644 --- a/scripts/quality/changes.mts +++ b/scripts/quality/changes.mts @@ -14,6 +14,35 @@ export const WORKSPACES = [ { name: 'chat', path: 'packages/chat', config: 'jest.config.js' }, ]; +export const REPORT_WORKSPACES = ['data-schemas', 'api', 'server']; + +export function reportSelection(check: string, scope: string) { + if (!['risk', 'mutation', 'both'].includes(check)) + throw new Error('Check must be risk, mutation, or both'); + const supported = check === 'mutation' ? ['data-schemas'] : REPORT_WORKSPACES; + const scopes = scope === 'all' ? supported : scope.split(','); + if ( + !scopes.length || + new Set(scopes).size !== scopes.length || + scopes.some((name) => !supported.includes(name)) + ) { + throw new Error(`Supported scopes for ${check}: ${supported.join(',')}, or all`); + } + return { check, scopes: REPORT_WORKSPACES.filter((name) => scopes.includes(name)) }; +} + +/** Risk reports cover backend workspaces; mutation shards only cover data-schemas. */ +export function reportJobs(workspaces: string[], check = 'both') { + return workspaces + .filter((name) => REPORT_WORKSPACES.includes(name)) + .flatMap((workspace) => [ + ...(check !== 'mutation' ? [{ workspace, task: { check: 'risk', shard: 0 } }] : []), + ...(workspace === 'data-schemas' && check !== 'risk' + ? [1, 2, 3, 4].map((shard) => ({ workspace, task: { check: 'mutation', shard } })) + : []), + ]); +} + export interface ChangedRange { start: number; count: number; @@ -125,22 +154,39 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur base: { type: 'string', default: 'origin/dev' }, head: { type: 'string', default: 'HEAD' }, scope: { type: 'string', default: 'all' }, + check: { type: 'string', default: 'both' }, }, }); - if (values.scope !== 'all' && !WORKSPACES.some(({ name }) => name === values.scope)) { - throw new Error(`Unknown workspace: ${values.scope}`); - } + const request = reportSelection(values.check, values.scope); + const selection = changes(values.base, values.head); const workspaces = WORKSPACES.filter( ({ name, path }) => - (values.scope === 'all' || values.scope === name) && + REPORT_WORKSPACES.includes(name) && + request.scopes.includes(name) && selection.files.some(({ file }) => workspaceFor(file)?.path === path), ).map(({ name }) => name); - console.log(JSON.stringify({ ...selection, workspaces }, null, 2)); + const reports = reportJobs(workspaces, request.check); + const mutationWorkspaces = [ + ...new Set( + reports.filter(({ task }) => task.check === 'mutation').map(({ workspace }) => workspace), + ), + ]; + const riskWorkspaces = reports + .filter(({ task }) => task.check === 'risk') + .map(({ workspace }) => workspace); + console.log(JSON.stringify({ ...selection, workspaces, reports, mutationWorkspaces }, null, 2)); if (process.env.GITHUB_OUTPUT) { appendFileSync( process.env.GITHUB_OUTPUT, - `workspaces=${JSON.stringify(workspaces)}\nhasChanges=${workspaces.length > 0}\n`, + [ + `workspaces=${JSON.stringify(workspaces)}`, + `hasChanges=${reports.length > 0}`, + `reports=${JSON.stringify(reports)}`, + `mutationWorkspaces=${JSON.stringify(mutationWorkspaces)}`, + `hasMutations=${mutationWorkspaces.length > 0}`, + '', + ].join('\n'), ); } if (process.env.GITHUB_STEP_SUMMARY) { @@ -151,7 +197,9 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur '', `Base: ${selection.base}. Head: ${selection.head}. Scope: ${values.scope}.`, '', - `Selected workspaces: ${workspaces.join(', ') || 'none'}.`, + `Change-risk workspaces: ${riskWorkspaces.join(', ') || 'none'}.`, + '', + `Mutation workspaces: ${mutationWorkspaces.join(', ') || 'none'} (four shards each).`, '', `Workspaces outside this run's scope: ${ WORKSPACES.filter( diff --git a/scripts/quality/changes.test.mts b/scripts/quality/changes.test.mts index 4fcbafa6a5..1f953849ab 100644 --- a/scripts/quality/changes.test.mts +++ b/scripts/quality/changes.test.mts @@ -93,9 +93,19 @@ test('assertion-only contract edits skip report jobs while a production edit rem }, ); assert.equal(run.status, 0, run.stderr); - return readFileSync(output, 'utf8'); + return Object.fromEntries( + readFileSync(output, 'utf8') + .trim() + .split('\n') + .map((line) => { + const index = line.indexOf('='); + return [line.slice(0, index), line.slice(index + 1)]; + }), + ); }; - assert.equal(select(head), 'workspaces=[]\nhasChanges=false\n'); + assert.equal(select(head).hasChanges, 'false'); + assert.equal(select(head).reports, '[]'); + assert.equal(select(head).hasMutations, 'false'); write(production, 'export function getFavorites() { return ["new"]; }\n'); const mixedHead = commit(); @@ -103,5 +113,81 @@ test('assertion-only contract edits skip report jobs while a production edit rem changes(base, mixedHead, directory).files.map(({ file }) => file), [production], ); - assert.equal(select(mixedHead), 'workspaces=["data-schemas"]\nhasChanges=true\n'); + assert.equal(select(mixedHead).workspaces, '["data-schemas"]'); + assert.equal(select(mixedHead).hasChanges, 'true'); + assert.equal(select(mixedHead).hasMutations, 'true'); }); + +for (const names of [ + [], + ['data-schemas'], + ['api'], + ['server'], + ['client'], + ['data-schemas', 'api', 'server', 'client'], +]) { + test(`report jobs select only changed backend owners: ${names.join(', ') || 'none'}`, (t) => { + const directory = mkdtempSync(join(tmpdir(), 'quality-report-selection-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + git(['init', '-q'], directory); + git(['config', 'user.name', 'Quality Test'], directory); + git(['config', 'user.email', 'quality@example.invalid'], directory); + git(['commit', '--allow-empty', '--no-gpg-sign', '-qm', 'base'], directory); + const base = git(['rev-parse', 'HEAD'], directory).trim(); + const paths: Record = { + 'data-schemas': 'packages/data-schemas/src/feature.ts', + api: 'packages/api/src/feature.ts', + server: 'api/server/feature.js', + client: 'client/src/feature.ts', + }; + for (const name of names) { + const file = join(directory, paths[name]); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, 'export function feature() { return 1; }'); + } + git(['add', '.'], directory); + git(['commit', '--allow-empty', '--no-gpg-sign', '-qm', 'head'], directory); + const select = (scope = 'all') => { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./changes.mts', import.meta.url)), + '--base', + base, + '--scope', + scope, + ], + { + cwd: directory, + encoding: 'utf8', + env: { ...process.env, GITHUB_OUTPUT: '', GITHUB_STEP_SUMMARY: '' }, + }, + ); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + const selected = select(); + const risk = names.filter((name) => name !== 'client'); + assert.deepEqual(selected.workspaces, risk); + assert.deepEqual( + selected.reports.filter(({ task }: { task: { check: string } }) => task.check === 'risk'), + risk.map((workspace) => ({ workspace, task: { check: 'risk', shard: 0 } })), + ); + const mutations = names.includes('data-schemas') ? ['data-schemas'] : []; + assert.deepEqual(selected.mutationWorkspaces, mutations); + assert.deepEqual( + selected.reports.filter(({ task }: { task: { check: string } }) => task.check === 'mutation'), + mutations.flatMap((workspace) => + [1, 2, 3, 4].map((shard) => ({ workspace, task: { check: 'mutation', shard } })), + ), + ); + for (const scope of ['data-schemas', 'api', 'server']) { + const narrowed = select(scope); + assert.deepEqual(narrowed.workspaces, names.includes(scope) ? [scope] : []); + assert.ok( + narrowed.reports.every(({ workspace }: { workspace: string }) => workspace === scope), + ); + if (scope !== 'data-schemas') assert.deepEqual(narrowed.mutationWorkspaces, []); + } + }); +} diff --git a/scripts/quality/comment.mts b/scripts/quality/comment.mts new file mode 100644 index 0000000000..1494519d53 --- /dev/null +++ b/scripts/quality/comment.mts @@ -0,0 +1,592 @@ +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { appendFileSync, readFileSync } from 'node:fs'; + +import { reportSelection } from './changes.mts'; + +export const GATE = 'Quality decision'; +const PREFIX = '')) return; + try { + const value: Request = JSON.parse(line.slice(PREFIX.length, -4)); + if ( + ![value.request, value.pr, value.run, value.attempt].every(positive) || + !SHA.test(value.head) || + !SHA.test(value.base) || + !/^[a-z\d-]+$/iu.test(value.actor) || + !['queued', 'running', 'completed', 'ignored'].includes(value.phase) || + typeof value.staticMutants !== 'boolean' || + !Array.isArray(value.scopes) || + !( + value.conclusion === null || + [ + 'success', + 'failure', + 'cancelled', + 'timed_out', + 'action_required', + 'neutral', + 'skipped', + 'stale', + 'startup_failure', + ].includes(value.conclusion) + ) + ) + return; + if (value.check === 'ignore') { + if (value.scopes.length || value.phase !== 'ignored') return; + } else { + const selection = reportSelection(value.check, value.scopes.join(',')); + if (JSON.stringify(selection.scopes) !== JSON.stringify(value.scopes)) return; + } + return value; + } catch { + return; + } +} + +async function pages(api: API, path: string, key?: string): Promise { + const all: T[] = []; + for (let page = 1; ; page++) { + const result = await api.request>( + 'GET', + `${path}${path.includes('?') ? '&' : '?'}per_page=100&page=${page}`, + ); + const items = key ? (result as Record)[key] : (result as T[]); + all.push(...items); + if (items.length < 100) return all; + } +} +const comments = (api: API, pr: number) => pages(api, `issues/${pr}/comments`); +const runURL = (context: Context, run: number) => + `${context.server}/${context.repository}/actions/runs/${run}`; + +class GitHubError extends Error { + readonly status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +function matchesRequest(run: Run, request: Request, context: Context) { + return ( + run.repository?.full_name.toLowerCase() === context.repository.toLowerCase() && + run.id === request.run && + run.event === 'issue_comment' && + run.path === '.github/workflows/change-quality.yml' && + run.display_title === `Quality PR #${request.pr} request ${request.request}` + ); +} + +function requests(items: Comment[], pr: number) { + return items.flatMap((item) => { + const request = readRequest(item); + return request?.pr === pr ? [request] : []; + }); +} + +async function activeRequests(api: API, context: Context, records: Request[]) { + const found = await Promise.all( + records + .filter((request) => request.check !== 'ignore') + .map(async (request) => { + try { + const run = await api.request('GET', `actions/runs/${request.run}`); + return matchesRequest(run, request, context) && run.status !== 'completed' + ? { request, run } + : undefined; + } catch (error) { + if (error instanceof GitHubError && error.status === 404) return; + throw error; + } + }), + ); + return found.filter((item) => item !== undefined); +} + +/** Force cancellation also stops jobs whose conditions use always(). */ +async function cancelRequest(api: API, context: Context, request: Request, run: Run) { + if (!matchesRequest(run, request, context) || run.status === 'completed') return; + try { + await api.request('POST', `actions/runs/${run.id}/force-cancel`); + } catch (error) { + // A run can finish between inspection and cancellation. + const current = await api.request('GET', `actions/runs/${run.id}`); + if (matchesRequest(current, request, context) && current.status === 'completed') return; + throw error; + } + for (let attempt = 0; attempt < 30; attempt++) { + const current = await api.request('GET', `actions/runs/${run.id}`); + if (!matchesRequest(current, request, context)) throw new Error('Quality run identity changed'); + if (current.status === 'completed') return; + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + throw new Error('Previous quality run has not stopped; retry the request'); +} + +export function renderRequest( + request: Request, + context: Context, + currentHead: string, + details = '', +) { + const scopes = request.scopes.join(', ') || 'none'; + const url = runURL(context, request.run); + const risk = request.check === 'risk' || request.check === 'both' ? scopes : 'none'; + const mutation = + request.check !== 'risk' && request.scopes.includes('data-schemas') + ? 'data-schemas (four shards)' + : 'none'; + return [ + `${PREFIX}${JSON.stringify(request)} -->`, + '### Quality request', + '', + ...(currentHead !== request.head + ? ['**Outdated: this request is for an earlier PR commit.**', ''] + : []), + `Commit: \`${request.head}\`. Requested by @${request.actor}.`, + `Checks: **${request.check === 'both' ? 'check-all' : request.check}**. Scopes: **${scopes}**. Static: **${request.staticMutants ? 'on' : 'off'}**${request.check === 'risk' || request.check === 'ignore' ? ' (not used)' : ''}.`, + `Risk: ${risk}. Mutation: ${mutation}.`, + ...(request.check === 'mutation' || request.check === 'both' + ? [ + 'Mutation testing is experimental. Only `data-schemas` is supported today. Support for more scopes is planned.', + ] + : []), + '', + request.check === 'ignore' + ? 'Quality checks explicitly ignored for this commit.' + : `Status: **${request.phase}${request.conclusion ? ` (${request.conclusion})` : ''}**. [Actions run](${url}) (attempt ${request.attempt}).`, + ...(details ? ['', details] : []), + '', + 'Scores are advisory. Failed tests, tool errors and incomplete reports do not clear the gate.', + ].join('\n'); +} + +export function gateDecision(records: Request[], head: string) { + const latest = records + .filter((item) => item.head === head) + .sort((a, b) => b.request - a.request)[0]; + if (!latest) + return { + state: 'pending', + description: 'Comment /quality check-all to run checks, or /quality ignore if not needed', + request: undefined, + }; + if (latest.check === 'ignore') + return { + state: 'success', + description: `Explicitly ignored by ${latest.actor}`, + request: latest, + }; + if (latest.phase !== 'completed') + return { + state: 'pending', + description: + 'Checks queued or running. Wait for results, or comment /quality ignore if not needed', + request: latest, + }; + return latest.conclusion === 'success' + ? { + state: 'success', + description: 'Selected quality checks completed; scores are advisory', + request: latest, + } + : { + state: 'failure', + description: + 'Checks failed or cancelled. Comment /quality check-all to retry, or /quality ignore if not needed', + request: latest, + }; +} + +async function updateGate(api: API, context: Context, pr: number) { + const pull = await api.request('GET', `pulls/${pr}`); + const records = (await comments(api, pr)).flatMap((comment) => { + const record = readRequest(comment); + return record?.pr === pr ? [record] : []; + }); + const decision = gateDecision(records, pull.head.sha); + await api.request('POST', `statuses/${pull.head.sha}`, { + context: GATE, + state: decision.state, + description: decision.description, + target_url: decision.request + ? runURL(context, decision.request.run) + : `${context.server}/${context.repository}/pull/${pr}`, + }); +} + +/** Authorization precedes parsing or any expensive work. No comment text is executed. */ +export async function acceptRequest(api: API, context: Context, pr: number, comment: Comment) { + if (comment.user.type === 'Bot' || !/^\/quality(?:\s|$)/u.test(comment.body.trim())) return; + const permission = await api.request<{ + permission: string; + user?: { permissions?: { push?: boolean } }; + }>('GET', `collaborators/${encodeURIComponent(comment.user.login)}/permission`); + if ( + !permission.user?.permissions?.push && + !['admin', 'maintain', 'write'].includes(permission.permission) + ) + return; + const pull = await api.request('GET', `pulls/${pr}`); + if (pull.state !== 'open') return; + let command: Command; + try { + command = parseCommand(comment.body)!; + } catch (error) { + await api.request('POST', `issues/${pr}/comments`, { + body: `Quality request was not started.\n\n${error instanceof Error ? error.message : usage}`, + }); + return; + } + const items = await comments(api, pr); + const records = requests(items, pr); + const latest = records.toSorted((a, b) => b.request - a.request)[0]; + const existing = items.find((item) => readRequest(item)?.request === comment.id); + const previous = existing && readRequest(existing); + if ((previous && previous.head !== pull.head.sha) || (latest && latest.request > comment.id)) { + throw new Error('This quality request was superseded. Post a new /quality command.'); + } + try { + await api.request('POST', `issues/comments/${comment.id}/reactions`, { content: 'eyes' }); + } catch { + console.warn('Could not add the quality request reaction. Continuing the request.'); + } + const active = await activeRequests(api, context, records); + const duplicate = active.find( + ({ request }) => + request.request === latest?.request && + request.run !== context.run && + request.head === pull.head.sha && + request.check === command.check && + request.staticMutants === command.staticMutants && + JSON.stringify(request.scopes) === JSON.stringify(command.scopes), + ); + if (duplicate) { + await api.request('POST', `issues/${pr}/comments`, { + body: `This quality request is already running for this commit. [Follow the existing run](${runURL(context, duplicate.run.id)}).`, + }); + return; + } + // Re-running Actions preserves the original request's commit and options. + const request: Request = { + ...(previous ?? { + ...command, + request: comment.id, + pr, + head: pull.head.sha, + base: pull.base.sha, + actor: comment.user.login, + }), + run: context.run, + attempt: context.attempt, + phase: (previous ?? command).check === 'ignore' ? 'ignored' : 'queued', + conclusion: null, + }; + if (!SHA.test(request.head) || !SHA.test(request.base)) throw new Error('Invalid PR commit'); + if (request.check === 'ignore') { + for (const prior of active) { + if (prior.run.id !== context.run) await cancelRequest(api, context, prior.request, prior.run); + } + } + const body = renderRequest(request, context, pull.head.sha); + const response = await api.request( + existing ? 'PATCH' : 'POST', + existing ? `issues/comments/${existing.id}` : `issues/${pr}/comments`, + { body }, + ); + if (request.check !== 'ignore') await updateGate(api, context, pr); + if (request.check !== 'ignore') { + for (const prior of active) { + if (prior.run.id !== context.run) await cancelRequest(api, context, prior.request, prior.run); + } + const current = await api.request('GET', `pulls/${pr}`); + if (current.head.sha !== request.head) + throw new Error('PR head changed. Post a new /quality command.'); + } + if (request.check === 'ignore') await updateGate(api, context, pr); + return { request, comment: response.id }; +} + +/** Refresh from GitHub's job metadata, never by executing or rendering artifact contents. */ +export async function publishRun(api: API, context: Context, run: Run) { + const match = /^Quality PR #(\d+) request (\d+)$/u.exec(run.display_title); + if ( + run.event !== 'issue_comment' || + run.path !== '.github/workflows/change-quality.yml' || + !match + ) + return; + const pr = Number(match[1]); + const source = Number(match[2]); + const comment = (await comments(api, pr)).find((item) => { + const record = readRequest(item); + return ( + record?.request === source && + record.pr === pr && + record.run === run.id && + record.attempt <= run.run_attempt + ); + }); + if (!comment) return; + const request = readRequest(comment)!; + if (!matchesRequest(run, request, context)) return; + const pull = await api.request('GET', `pulls/${pr}`); + const latest = requests(await comments(api, pr), pr).toSorted((a, b) => b.request - a.request)[0]; + if ( + run.status !== 'completed' && + (request.head !== pull.head.sha || latest.request > request.request) + ) { + await cancelRequest(api, context, request, run); + run = await api.request('GET', `actions/runs/${run.id}`); + } + if (request.check === 'ignore') { + await updateGate(api, context, pr); + return; + } + const jobs = await pages(api, `actions/runs/${run.id}/jobs?filter=all`, 'jobs'); + // Failed-job reruns reuse successful jobs from earlier attempts. + const latestJobs = new Map(); + for (const job of jobs) { + if ((latestJobs.get(job.name)?.id ?? 0) < job.id) latestJobs.set(job.name, job); + } + const reports = [...latestJobs.values()].filter(({ name }) => + /^(?:Change risk|Mutation testing [1-4]\/4|Mutation report) \((?:data-schemas|api|server)\)$/u.test( + name, + ), + ); + const artifacts = await pages(api, `actions/runs/${run.id}/artifacts`, 'artifacts'); + const links = artifacts.filter( + ({ name, expired }) => + !expired && + new RegExp( + `^change-(?:risk|mutation)-(?:data-schemas|api|server)-${request.head}(?:-shard-[1-4])?$`, + 'u', + ).test(name), + ); + // A delayed progress event must not replace a terminal report. + if ( + request.attempt === run.run_attempt && + request.phase === 'completed' && + run.status !== 'completed' + ) + return; + request.attempt = run.run_attempt; + request.phase = run.status === 'completed' ? 'completed' : 'running'; + const checksSucceeded = + ['Select changed source', 'Verify quality tooling'].every( + (name) => latestJobs.get(name)?.conclusion === 'success', + ) && reports.every((job) => job.conclusion === 'success'); + request.conclusion = + run.status !== 'completed' + ? null + : run.conclusion === 'success' && !checksSucceeded + ? 'failure' + : run.conclusion; + const details = [ + ...(reports.length + ? [ + '| Check | Result |', + '| --- | --- |', + ...reports.map( + (job) => + `| [${job.name}](${runURL(context, run.id)}/job/${job.id}) | ${job.conclusion ?? job.status} |`, + ), + ] + : [ + 'No report jobs have started. A successful empty selection means no supported source changed.', + ]), + '', + ...(links.length + ? [ + 'Reports and scores:', + ...links.map( + (artifact, index) => + `${index + 1}. [${artifact.name}](${runURL(context, run.id)}/artifacts/${artifact.id})`, + ), + ] + : ['No report artifacts are available.']), + ].join('\n'); + await api.request('PATCH', `issues/comments/${comment.id}`, { + body: renderRequest(request, context, pull.head.sha, details), + }); + await updateGate(api, context, pr); +} + +export async function refreshPull(api: API, context: Context, pr: number) { + const pull = await api.request('GET', `pulls/${pr}`); + const items = await comments(api, pr); + for (const prior of await activeRequests(api, context, requests(items, pr))) { + if (prior.request.head !== pull.head.sha) + await cancelRequest(api, context, prior.request, prior.run); + } + for (const comment of items) { + const request = readRequest(comment); + if (!request || request.pr !== pr) continue; + // Keep results and artifact links, including when a force push restores the same commit. + const stale = '\n**Outdated: this request is for an earlier PR commit.**\n'; + const clean = comment.body.replace(stale, ''); + const body = + request.head === pull.head.sha + ? clean + : clean.replace('### Quality request\n', `### Quality request\n${stale}`); + if (body === comment.body) continue; + await api.request('PATCH', `issues/comments/${comment.id}`, { body }); + } + await updateGate(api, context, pr); +} + +export function client(env: NodeJS.ProcessEnv): API { + return { + async request(method: string, path: string, body?: object): Promise { + const response = await fetch( + `${env.GITHUB_API_URL ?? 'https://api.github.com'}${path === '/graphql' ? path : `/repos/${env.GITHUB_REPOSITORY}/${path}`}`, + { + method, + headers: { + accept: 'application/vnd.github+json', + authorization: `Bearer ${env.GH_TOKEN}`, + 'content-type': 'application/json', + 'x-github-api-version': '2022-11-28', + }, + body: body ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(30000), + }, + ); + if (!response.ok) + throw new GitHubError( + response.status, + `GitHub ${method} ${path.split('?')[0]} failed: HTTP ${response.status}`, + ); + const text = await response.text(); + return (text ? JSON.parse(text) : undefined) as T; + }, + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const env = process.env; + const api = client(env); + const context = { + repository: env.GITHUB_REPOSITORY!, + server: env.GITHUB_SERVER_URL!, + run: Number(env.GITHUB_RUN_ID), + attempt: Number(env.GITHUB_RUN_ATTEMPT), + }; + const event = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH!, 'utf8')); + if (process.argv[2] === 'request' && event.issue?.pull_request) { + const accepted = await acceptRequest(api, context, event.issue.number, event.comment); + if (accepted && env.GITHUB_OUTPUT) { + const { request } = accepted; + appendFileSync( + env.GITHUB_OUTPUT, + Object.entries({ + accepted: request.check !== 'ignore', + head: request.head, + base: request.base, + scope: request.scopes.join(','), + check: request.check, + static: request.staticMutants ? 'on' : 'off', + }) + .map(([key, value]) => `${key}=${value}\n`) + .join(''), + ); + } + } else if (process.argv[2] === 'progress') { + await publishRun(api, context, await api.request('GET', `actions/runs/${context.run}`)); + } else if (env.GITHUB_EVENT_NAME === 'workflow_run') { + // Read the current attempt so a late event cannot replace a newer rerun's result. + await publishRun( + api, + context, + await api.request('GET', `actions/runs/${event.workflow_run.id}`), + ); + } else if (env.GITHUB_EVENT_NAME === 'pull_request_target') { + await refreshPull(api, context, event.pull_request.number); + } +} diff --git a/scripts/quality/comment.test.mts b/scripts/quality/comment.test.mts new file mode 100644 index 0000000000..d926ce1003 --- /dev/null +++ b/scripts/quality/comment.test.mts @@ -0,0 +1,676 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; + +import type { API, Comment, Context, Request } from './comment.mts'; + +import { + acceptRequest, + gateDecision, + parseCommand, + publishRun, + readRequest, + refreshPull, + renderRequest, +} from './comment.mts'; +import { reportJobs, reportSelection } from './changes.mts'; + +const head = 'a'.repeat(40); +const base = 'b'.repeat(40); +const next = 'c'.repeat(40); +const context: Context = { + repository: 'owner/repo', + server: 'https://github.com', + run: 55, + attempt: 1, +}; +const command = (id = 10, body = '/quality both scopes=all static=off'): Comment => ({ + id, + body, + user: { login: 'maintainer', type: 'User' }, +}); +const record: Request = { + request: 10, + pr: 1, + head, + base, + run: 55, + attempt: 1, + actor: 'maintainer', + phase: 'queued', + conclusion: null, + check: 'both', + scopes: ['data-schemas', 'api', 'server'], + staticMutants: false, +}; +const bot = (body: string, id = 100): Comment => ({ + id, + body, + user: { login: 'github-actions[bot]', type: 'Bot' }, +}); +const prerequisiteJobs = [ + { name: 'Accept quality request', conclusion: 'success', status: 'completed', id: 70 }, + { name: 'Select changed source', conclusion: 'success', status: 'completed', id: 71 }, + { name: 'Verify quality tooling', conclusion: 'success', status: 'completed', id: 72 }, +]; +const run = (status = 'completed', conclusion: string | null = 'success') => ({ + id: 55, + repository: { full_name: context.repository }, + run_attempt: 1, + event: 'issue_comment', + path: '.github/workflows/change-quality.yml', + display_title: 'Quality PR #1 request 10', + status, + conclusion, +}); + +class GitHub implements API { + permission = 'write'; + reactionError = false; + runs = new Map>([[55, run()]]); + head = head; + state = 'open'; + comments: Comment[] = []; + statuses: { sha: string; state: string; description: string }[] = []; + calls: { method: string; path: string; body?: object }[] = []; + jobs = [ + ...structuredClone(prerequisiteJobs), + { name: 'Mutation report (data-schemas)', conclusion: 'success', status: 'completed', id: 90 }, + ]; + artifacts = [{ id: 91, name: `change-mutation-data-schemas-${head}`, expired: false }]; + async request(method: string, path: string, body?: object): Promise { + this.calls.push({ method, path, body }); + let result: object; + if (path.endsWith('/permission')) result = { permission: this.permission }; + else if (path === 'pulls/1') + result = { number: 1, state: this.state, head: { sha: this.head }, base: { sha: base } }; + else if (method === 'GET' && path.startsWith('issues/1/comments?')) { + const page = Number(new URLSearchParams(path.split('?')[1]).get('page')); + result = this.comments.slice((page - 1) * 100, page * 100); + } else if (method === 'POST' && path === 'issues/1/comments') { + const comment = bot((body as { body: string }).body, 10000 + this.comments.length); + this.comments.push(comment); + result = comment; + } else if (method === 'POST' && /^issues\/comments\/\d+\/reactions$/.test(path)) { + if (this.reactionError) throw new Error('Reaction unavailable'); + result = { content: 'eyes' }; + } else if (method === 'PATCH' && path.startsWith('issues/comments/')) { + const comment = this.comments.find(({ id }) => id === Number(path.split('/').at(-1)))!; + comment.body = (body as { body: string }).body; + result = comment; + } else if (method === 'POST' && path.startsWith('statuses/')) { + this.statuses.push({ + sha: path.slice(9), + ...(body as { state: string; description: string }), + }); + result = {}; + } else if (method === 'POST' && path.endsWith('/force-cancel')) { + const current = this.runs.get(Number(path.split('/')[2]))!; + current.status = 'completed'; + current.conclusion = 'cancelled'; + result = {}; + } else if (method === 'GET' && /^actions\/runs\/\d+$/.test(path)) { + result = this.runs.get(Number(path.split('/')[2])) ?? { + ...run(), + id: Number(path.split('/')[2]), + }; + } else if (path.includes('/jobs?')) result = { jobs: this.jobs }; + else if (path.includes('/artifacts?')) result = { artifacts: this.artifacts }; + else throw new Error(`Unexpected API call: ${method} ${path}`); + return structuredClone(result) as T; + } +} + +test('commands validate choices and lists as data, with all scopes and static off by default', () => { + assert.deepEqual(parseCommand('/quality mutation'), { + check: 'mutation', + scopes: ['data-schemas'], + staticMutants: false, + }); + assert.deepEqual(parseCommand('/quality risk scopes=server,api'), { + check: 'risk', + scopes: ['api', 'server'], + staticMutants: false, + }); + assert.deepEqual(parseCommand('/quality both scopes=data-schemas,server static=off'), { + check: 'both', + scopes: ['data-schemas', 'server'], + staticMutants: false, + }); + assert.deepEqual(parseCommand('/quality check-all'), { + check: 'both', + scopes: ['data-schemas', 'api', 'server'], + staticMutants: false, + }); + assert.deepEqual( + parseCommand('/quality check-all'), + parseCommand('/quality both scopes=all static=off'), + ); + assert.deepEqual(parseCommand('/quality check-all scopes=data-schemas static=on'), { + check: 'both', + scopes: ['data-schemas'], + staticMutants: true, + }); + assert.equal(parseCommand('/quality ignore')?.check, 'ignore'); + assert.equal(parseCommand('ordinary comment'), undefined); + for (const body of [ + '/quality', + '/quality check-all static=invalid', + '/quality check-all scopes=client', + '/quality mutation scopes=api', + '/quality both scopes=all,api', + '/quality risk scopes=api,api', + '/quality both static=no', + '/quality both static=off static=on', + '/quality ignore scopes=all', + '/quality both $(id)', + '/quality both scopes=api;id', + '/quality both\nGITHUB_OUTPUT=bad', + '/quality both unknown=x', + '/quality both scopes=', + '/quality both scopes=client', + ]) { + assert.throws(() => parseCommand(body), Error, body); + } + for (const check of ['risk', 'mutation', 'both']) { + const selected = reportSelection(check, 'all'); + const jobs = reportJobs(selected.scopes, check); + assert.equal( + jobs.filter(({ task }) => task.check === 'risk').length, + check === 'mutation' ? 0 : 3, + ); + assert.equal( + jobs.filter(({ task }) => task.check === 'mutation').length, + check === 'risk' ? 0 : 4, + ); + assert.ok( + jobs.every( + ({ task, workspace }) => task.check !== 'mutation' || workspace === 'data-schemas', + ), + ); + } +}); + +test('read access, triage, bots and closed PRs cannot start work or clear the gate', async () => { + for (const permission of ['read', 'triage', 'none']) { + const api = new GitHub(); + api.permission = permission; + assert.equal(await acceptRequest(api, context, 1, command()), undefined); + assert.equal(api.comments.length, 0); + assert.equal(api.statuses.length, 0); + assert.equal(api.calls.length, 1); + } + const api = new GitHub(); + await acceptRequest(api, context, 1, { ...command(), user: bot('').user }); + assert.equal(api.calls.length, 0); + api.state = 'closed'; + assert.equal(await acceptRequest(api, context, 1, command()), undefined); + assert.equal(api.comments.length, 0); +}); + +async function accept(api: GitHub, ctx: Context, pr: number, comment: Comment) { + const result = await acceptRequest(api, ctx, pr, comment); + if (result) await refreshPull(api, ctx, pr); + return result; +} + +test('one exact-head request comment moves through progress and completion with artifact links', async () => { + const api = new GitHub(); + const accepted = await accept(api, context, 1, command()); + assert.equal(accepted?.request.head, head); + assert.deepEqual( + api.calls.find(({ path }) => path.endsWith('/reactions')), + { + method: 'POST', + path: 'issues/comments/10/reactions', + body: { content: 'eyes' }, + }, + ); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + assert.match(api.comments[0].body, /Static: \*\*off\*\*/); + assert.match(api.comments[0].body, /actions\/runs\/55/); + await publishRun(api, context, run('in_progress', null)); + assert.match(api.comments[0].body, /\*\*running\*\*/); + await publishRun(api, context, run()); + assert.equal(api.comments.length, 1); + assert.match(api.comments[0].body, /completed \(success\)/); + assert.match(api.comments[0].body, /artifacts\/91/); + assert.equal(api.statuses.at(-1)?.state, 'success'); + await publishRun(api, context, run('in_progress', null)); + assert.match(api.comments[0].body, /completed \(success\)/); +}); + +test('failures and cancellations keep the gate closed, while explicit ignore clears it', async () => { + for (const conclusion of ['failure', 'cancelled', 'timed_out', 'action_required', 'skipped']) { + const api = new GitHub(); + await accept(api, context, 1, command()); + await publishRun(api, context, run('completed', conclusion)); + assert.equal(api.statuses.at(-1)?.state, 'failure'); + await accept(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')); + assert.equal(api.statuses.at(-1)?.state, 'success'); + assert.match(api.comments[1].body, /explicitly ignored/); + // A late result from the older request cannot undo the explicit ignore. + await publishRun(api, context, run('completed', conclusion)); + assert.equal(api.statuses.at(-1)?.state, 'success'); + } +}); + +test('new heads need a fresh decision; old results and ignore decisions are marked outdated', async () => { + const api = new GitHub(); + await accept(api, context, 1, command()); + await publishRun(api, context, run()); + api.head = next; + await refreshPull(api, context, 1); + assert.match(api.comments[0].body, /\*\*Outdated:/); + assert.match(api.comments[0].body, /artifacts\/91/); + assert.equal(api.statuses.at(-1)?.sha, next); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + await publishRun(api, context, run()); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + const accepted = await accept(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')); + assert.equal(accepted?.request.head, next); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('reruns of the current request preserve its commit; stale attempts cannot overwrite them', async () => { + const api = new GitHub(); + await accept(api, context, 1, command()); + const second = { ...context, attempt: 2 }; + await accept(api, second, 1, command()); + assert.equal(api.comments.length, 1); + assert.equal(readRequest(api.comments[0])?.head, head); + await publishRun(api, context, run()); + assert.equal(readRequest(api.comments[0])?.phase, 'queued'); + await publishRun(api, second, { ...run(), run_attempt: 2 }); + assert.equal(readRequest(api.comments[0])?.phase, 'completed'); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('latest explicit request owns the gate; automatic reports and spoofed records cannot clear it', async () => { + assert.equal(gateDecision([], head).state, 'pending'); + assert.equal( + gateDecision( + [ + { ...record, phase: 'completed', conclusion: 'success' }, + { ...record, request: 11 }, + ], + head, + ).state, + 'pending', + ); + const forged = renderRequest( + { ...record, check: 'ignore', scopes: [], phase: 'ignored' }, + context, + head, + ); + assert.equal(readRequest({ ...command(), body: forged }), undefined); + assert.equal(readRequest(bot(forged.replace(head, 'invalid'))), undefined); + const api = new GitHub(); + api.comments.push({ ...command(), body: forged }); + await refreshPull(api, context, 1); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + await accept(api, context, 1, command()); + const before = api.comments.at(-1)?.body; + for (const unrelated of [ + { ...run(), event: 'pull_request' }, + { ...run(), path: '.github/workflows/other.yml' }, + { ...run(), id: 57 }, + ]) { + await publishRun(api, context, unrelated); + assert.equal(api.comments.at(-1)?.body, before); + } +}); + +test('pagination finds old requests and excludes unrelated or expired artifacts', async () => { + const api = new GitHub(); + api.comments = Array.from({ length: 101 }, (_, i) => command(200 + i, 'ordinary comment')); + await accept(api, context, 1, command()); + api.artifacts.push( + { id: 92, name: 'untrusted [link](https://example.com)', expired: false }, + { id: 93, name: `change-risk-api-${head}`, expired: true }, + ); + await publishRun(api, context, run()); + assert.equal(api.comments.length, 102); + assert.doesNotMatch(api.comments.at(-1)!.body, /untrusted|artifacts\/93/); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('rerunning only failed jobs advances the attempt without reaccepting the command', async () => { + const api = new GitHub(); + await accept(api, context, 1, command()); + await publishRun(api, context, run('completed', 'failure')); + assert.equal(api.statuses.at(-1)?.state, 'failure'); + await publishRun(api, context, { ...run('in_progress', null), run_attempt: 2 }); + assert.equal(readRequest(api.comments[0])?.attempt, 2); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + await publishRun(api, context, { ...run(), run_attempt: 2 }); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('a rejected rerun cannot clear a failed decision with skipped reports', async () => { + const api = new GitHub(); + await accept(api, context, 1, command()); + await publishRun(api, context, run('completed', 'failure')); + api.permission = 'read'; + assert.equal(await acceptRequest(api, { ...context, attempt: 2 }, 1, command()), undefined); + api.jobs = [ + ...api.jobs, + ...prerequisiteJobs.map((job) => ({ + ...job, + id: job.id + 100, + conclusion: job.name === 'Accept quality request' ? 'success' : 'skipped', + })), + { name: 'Mutation report (data-schemas)', conclusion: 'skipped', status: 'completed', id: 190 }, + ]; + api.artifacts = []; + await publishRun(api, context, { ...run(), run_attempt: 2 }); + assert.equal(readRequest(api.comments[0])?.conclusion, 'failure'); + assert.equal(api.statuses.at(-1)?.state, 'failure'); + await refreshPull(api, context, 1); + assert.equal(api.statuses.at(-1)?.state, 'failure'); +}); + +test('successful workflows still require successful selection, tooling and selected reports', async () => { + for (const name of [ + 'Select changed source', + 'Verify quality tooling', + 'Mutation report (data-schemas)', + ]) { + for (const conclusion of ['skipped', 'failure', 'cancelled']) { + const api = new GitHub(); + await accept(api, context, 1, command()); + api.jobs.find((job) => job.name === name)!.conclusion = conclusion; + await publishRun(api, context, run()); + assert.equal(api.statuses.at(-1)?.state, 'failure', `${name}: ${conclusion}`); + } + const api = new GitHub(); + await accept(api, context, 1, command()); + if (name !== 'Mutation report (data-schemas)') { + api.jobs = api.jobs.filter((job) => job.name !== name); + await publishRun(api, context, run()); + assert.equal(api.statuses.at(-1)?.state, 'failure', `Missing ${name}`); + } + } +}); + +test('a verified empty selection needs no report jobs or artifacts', async () => { + const api = new GitHub(); + await accept(api, context, 1, command()); + api.jobs = structuredClone(prerequisiteJobs); + api.artifacts = []; + await publishRun(api, context, run()); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('completion recovers an ignore decision when its status job was cancelled', async () => { + const api = new GitHub(); + await acceptRequest(api, context, 1, command(10, '/quality ignore')); + api.statuses = []; + await publishRun(api, context, run('completed', 'cancelled')); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('restoring the original commit removes the outdated label', async () => { + const api = new GitHub(); + await accept(api, context, 1, command(10, '/quality ignore')); + api.head = next; + await refreshPull(api, context, 1); + assert.match(api.comments[0].body, /Outdated:/); + api.head = head; + await refreshPull(api, context, 1); + assert.doesNotMatch(api.comments[0].body, /Outdated:/); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('failed-job reruns show reused successful jobs and the latest retried outcome', async () => { + const api = new GitHub(); + await accept(api, context, 1, command()); + api.jobs = [ + ...structuredClone(prerequisiteJobs), + { name: 'Mutation report (data-schemas)', status: 'completed', conclusion: 'failure', id: 89 }, + { name: 'Change risk (api)', status: 'completed', conclusion: 'success', id: 88 }, + { name: 'Mutation report (data-schemas)', status: 'completed', conclusion: 'success', id: 90 }, + ]; + await publishRun(api, context, { ...run(), run_attempt: 2 }); + assert.match(api.comments[0].body, /job\/88/); + assert.match(api.comments[0].body, /job\/90/); + assert.doesNotMatch(api.comments[0].body, /job\/89/); +}); + +function running(api: GitHub, value: Request = record, overrides = {}) { + api.comments.push(bot(renderRequest(value, context, value.head), 100 + api.comments.length)); + api.runs.set(value.run, { + ...run('in_progress', null), + id: value.run, + display_title: `Quality PR #${value.pr} request ${value.request}`, + ...overrides, + }); +} +const cancellations = (api: GitHub) => + api.calls.filter(({ path }) => path.endsWith('/force-cancel')); + +test('a duplicate active command links the existing run without changing its decision', async () => { + const api = new GitHub(); + running(api); + const original = api.comments[0].body; + const accepted = await acceptRequest( + api, + { ...context, run: 56 }, + 1, + command(11, '/quality check-all'), + ); + assert.equal(accepted, undefined); + assert.equal(api.comments[0].body, original); + assert.match(api.comments[1].body, /already running.*actions\/runs\/55/); + assert.equal(readRequest(api.comments[1]), undefined); + assert.equal(cancellations(api).length, 0); + assert.equal(api.statuses.length, 0); +}); + +test('different validated options replace the active run only after recording acceptance', async () => { + const api = new GitHub(); + running(api); + const accepted = await acceptRequest( + api, + { ...context, run: 56 }, + 1, + command(11, '/quality both scopes=all static=on'), + ); + assert.equal(accepted?.request.staticMutants, true); + assert.deepEqual( + cancellations(api).map(({ path }) => path), + ['actions/runs/55/force-cancel'], + ); + assert.equal(api.runs.get(55)?.status, 'completed'); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + const acceptedAt = api.calls.findIndex( + ({ method, path }) => method === 'POST' && path === 'issues/1/comments', + ); + const cancelledAt = api.calls.findIndex(({ path }) => path.endsWith('/force-cancel')); + assert.ok(acceptedAt < cancelledAt); +}); + +test('a finished request can start a fresh run with the same options', async () => { + const api = new GitHub(); + running(api, { ...record, phase: 'completed', conclusion: 'success' }, run()); + const accepted = await acceptRequest(api, { ...context, run: 56 }, 1, command(11)); + assert.equal(accepted?.request.run, 56); + assert.equal(cancellations(api).length, 0); + assert.equal(api.statuses.at(-1)?.state, 'pending'); +}); + +test('ignore stops accepted quality work before clearing the gate', async () => { + const api = new GitHub(); + running(api); + await acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')); + assert.equal(cancellations(api).length, 1); + const stoppedAt = api.calls.findIndex(({ path }) => path.endsWith('/force-cancel')); + const clearedAt = api.calls.findIndex(({ path }) => path.startsWith('statuses/')); + assert.ok(stoppedAt < clearedAt); + assert.equal(api.statuses.at(-1)?.state, 'success'); +}); + +test('invalid commands and unauthorized actors cannot disturb an active request', async () => { + for (const [permission, body] of [ + ['read', '/quality ignore'], + ['triage', '/quality both static=on'], + ['write', '/quality mutation scopes=server'], + ['write', '/quality both static=off static=on'], + ]) { + const api = new GitHub(); + running(api); + api.permission = permission; + const original = api.comments[0].body; + assert.equal( + await acceptRequest(api, { ...context, run: 56 }, 1, command(11, body)), + undefined, + ); + assert.equal(api.comments[0].body, original); + assert.equal(api.runs.get(55)?.status, 'in_progress'); + assert.equal(cancellations(api).length, 0); + assert.equal(api.statuses.length, 0); + assert.ok(api.calls.every(({ path }) => !path.endsWith('/reactions'))); + } +}); + +test('cancellation excludes other repositories, PRs, workflows and standalone manual runs', async () => { + for (const overrides of [ + { repository: { full_name: 'other/repo' } }, + { display_title: 'Quality PR #2 request 10' }, + { display_title: 'Quality PR #1 request 999' }, + { event: 'workflow_dispatch' }, + { event: 'pull_request' }, + { path: '.github/workflows/unrelated.yml' }, + { id: 999 }, + ]) { + const api = new GitHub(); + running(api, record, overrides); + await acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')); + assert.equal(cancellations(api).length, 0, JSON.stringify(overrides)); + } + const api = new GitHub(); + running(api, { ...record, pr: 2 }); + await acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')); + assert.equal(cancellations(api).length, 0); +}); + +test('a new head cancels old work and needs a fresh decision', async () => { + const api = new GitHub(); + running(api); + api.head = next; + await refreshPull(api, { ...context, run: 99 }, 1); + assert.equal(cancellations(api).length, 1); + assert.match(api.comments[0].body, /Outdated:/); + assert.equal(api.statuses.at(-1)?.sha, next); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + await assert.rejects(acceptRequest(api, { ...context, attempt: 2 }, 1, command()), /superseded/); +}); + +test('an old Actions rerun cannot replace a newer accepted ignore', async () => { + const api = new GitHub(); + running(api); + api.comments.push( + bot( + renderRequest( + { ...record, request: 11, run: 56, check: 'ignore', scopes: [], phase: 'ignored' }, + context, + head, + ), + 101, + ), + ); + await publishRun(api, { ...context, run: 99 }, { ...run('in_progress', null), run_attempt: 2 }); + assert.equal(cancellations(api).length, 1); + assert.equal(api.statuses.at(-1)?.state, 'success'); + assert.equal(readRequest(api.comments[0])?.conclusion, 'cancelled'); +}); + +test('delayed older comments cannot replace a newer decision', async () => { + const api = new GitHub(); + running(api, { ...record, request: 12 }); + await assert.rejects( + acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')), + /superseded/, + ); + assert.equal(cancellations(api).length, 0); + assert.equal(api.statuses.length, 0); +}); + +test('an older matching run does not hide different options in the current request', async () => { + const api = new GitHub(); + running(api); + running(api, { ...record, request: 11, run: 56, staticMutants: true }); + const accepted = await acceptRequest(api, { ...context, run: 57 }, 1, command(12)); + assert.equal(accepted?.request.run, 57); + assert.deepEqual( + cancellations(api).map(({ path }) => path), + ['actions/runs/55/force-cancel', 'actions/runs/56/force-cancel'], + ); +}); + +test('failed cancellation blocks the replacement instead of starting duplicate work', async () => { + const api = new GitHub(); + running(api); + const request = api.request.bind(api); + api.request = async (method: string, path: string, body?: object): Promise => { + if (path.endsWith('/force-cancel')) throw new Error('Cancellation unavailable'); + return request(method, path, body); + }; + await assert.rejects( + acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality both static=on')), + /Cancellation unavailable/, + ); + assert.equal(api.runs.get(55)?.status, 'in_progress'); + assert.equal(api.statuses.at(-1)?.state, 'pending'); +}); + +test('failed ignore cancellation cannot clear the gate through completion or refresh', async () => { + const api = new GitHub(); + running(api); + const call = api.request.bind(api); + api.request = async (method: string, path: string, body?: object): Promise => { + if (path.endsWith('/force-cancel')) throw new Error('Cancellation unavailable'); + return call(method, path, body); + }; + await assert.rejects( + acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality ignore')), + /Cancellation unavailable/, + ); + assert.equal(api.comments.length, 1); + assert.equal(api.runs.get(55)?.status, 'in_progress'); + await publishRun(api, context, { + ...run('completed', 'failure'), + id: 56, + display_title: 'Quality PR #1 request 11', + }); + await refreshPull(api, context, 1); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + assert.ok(api.statuses.every(({ state }) => state !== 'success')); +}); + +test('a head change during replacement prevents old-head report jobs from starting', async () => { + const api = new GitHub(); + running(api); + const request = api.request.bind(api); + api.request = async (method: string, path: string, body?: object): Promise => { + const response = await request(method, path, body); + if (path.endsWith('/force-cancel')) api.head = next; + return response; + }; + await assert.rejects( + acceptRequest(api, { ...context, run: 56 }, 1, command(11, '/quality both static=on')), + /PR head changed/, + ); + await refreshPull(api, { ...context, run: 99 }, 1); + assert.equal(api.statuses.at(-1)?.sha, next); + assert.equal(api.statuses.at(-1)?.state, 'pending'); +}); + +test('reaction failure does not stop an authorized quality request', async (t) => { + const warning = t.mock.method(console, 'warn', () => {}); + const api = new GitHub(); + api.reactionError = true; + const accepted = await acceptRequest(api, context, 1, command(10, '/quality check-all')); + assert.equal(accepted?.request.head, head); + assert.equal(api.statuses.at(-1)?.state, 'pending'); + assert.equal(warning.mock.callCount(), 1); +}); diff --git a/scripts/quality/jest.mts b/scripts/quality/jest.mts index 89f7bd0d73..5e83d83dc5 100644 --- a/scripts/quality/jest.mts +++ b/scripts/quality/jest.mts @@ -1,3 +1,7 @@ +import { createRequire } from 'node:module'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; + import { literalGlob } from './changes.mts'; /** Jest accepts one JSON string for collectCoverageFrom, not separate file arguments. */ @@ -6,12 +10,13 @@ export function coverageArgs( files: string[], output: string, tests: string[], + serial = false, ): string[] { return [ '--config', config, '--ci', - '--runInBand', + ...(serial ? ['--runInBand'] : []), '--forceExit', '--coverage', '--coverageProvider=babel', @@ -29,3 +34,80 @@ export function coverageArgs( ...tests, ]; } + +/** Redis integration suites use the same environment and ordering as the cache CI lane. */ +export function coverageGroups(workspace: string, tests: string[]) { + const groups = [ + { name: 'unit', tests: [] as string[], redis: false, serial: false }, + { name: 'cache-core', tests: [] as string[], redis: true, serial: false }, + { name: 'cache-cluster', tests: [] as string[], redis: true, serial: true }, + { name: 'cache-mcp', tests: [] as string[], redis: true, serial: false }, + { name: 'cache-stream', tests: [] as string[], redis: true, serial: true }, + ]; + for (const test of tests) { + const path = test.replaceAll('\\', '/'); + let group = 0; + if (workspace === 'api') { + if (path.includes('.stream_integration.')) group = 4; + else if (path.includes('.cache_integration.')) { + group = path.includes('/src/cluster/') ? 2 : path.includes('/src/mcp/') ? 3 : 1; + } + } + groups[group].tests.push(test); + } + return groups.filter(({ tests }) => tests.length); +} + +export function runCoverage(options: { + workspace: string; + config: string; + cwd: string; + jestBin: string; + files: string[]; + tests: string[]; + output: string; + env: NodeJS.ProcessEnv; +}) { + const { workspace, config, cwd, jestBin, files, tests, output, env } = options; + // Use the coverage merger installed with the measured head's Jest. + const { createCoverageMap } = createRequire(jestBin)( + 'istanbul-lib-coverage', + ) as typeof import('istanbul-lib-coverage'); + mkdirSync(output, { recursive: true }); + const combined = createCoverageMap({}); + const groups = coverageGroups(workspace, tests); + writeFileSync(`${output}/test-groups.json`, JSON.stringify(groups, null, 2)); + for (const group of groups) { + const directory = groups.length === 1 ? output : `${output}/${group.name}`; + mkdirSync(directory, { recursive: true }); + const reports = ['coverage-final.json', 'tests.json']; + for (const name of reports) rmSync(`${output}/${name}`, { force: true }); + console.log(`Coverage group: ${group.name}; ${group.tests.length} test files`); + const result = spawnSync( + process.execPath, + // Jest includes coverageDirectory in its transform-cache key. Keep it stable across groups. + [jestBin, ...coverageArgs(config, files, output, group.tests, group.serial)], + { + cwd, + stdio: 'inherit', + env: { ...env, ...(group.redis ? { USE_REDIS: 'true', USE_REDIS_CLUSTER: 'false' } : {}) }, + }, + ); + if (directory !== output) { + for (const name of reports) { + if (existsSync(`${output}/${name}`)) + renameSync(`${output}/${name}`, `${directory}/${name}`); + } + } + if (result.error || result.status !== 0) { + throw ( + result.error ?? + new Error(`Jest ${group.name} exited with ${result.status ?? result.signal}`) + ); + } + const file = `${directory}/coverage-final.json`; + if (!existsSync(file)) throw new Error('Jest did not produce coverage'); + combined.merge(JSON.parse(readFileSync(file, 'utf8'))); + } + writeFileSync(`${output}/coverage-final.json`, JSON.stringify(combined.toJSON())); +} diff --git a/scripts/quality/merge.mts b/scripts/quality/merge.mts new file mode 100644 index 0000000000..b66f46b5ea --- /dev/null +++ b/scripts/quality/merge.mts @@ -0,0 +1,178 @@ +import { parseArgs } from 'node:util'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; + +import type { Reporter } from '@stryker-mutator/api/report'; +import type { ShardPlan } from './shards.mts'; + +import { assertMutants, mutantKey } from './shards.mts'; + +export type MutationReport = Parameters>[0]; +export interface ShardResult { + plan: ShardPlan; + completion: { digest: string; index: number; complete: boolean; total: number }; + report?: MutationReport; +} + +const terminalStatuses = new Set([ + 'Killed', + 'Survived', + 'Timeout', + 'NoCoverage', + 'CompileError', + 'Ignored', +]); + +/** Only publish a score after every shard proves that its assigned mutations finished. */ +export function mergeReports(results: ShardResult[]) { + const plan = results[0]?.plan; + if (!plan || plan.shards.length !== 4 || results.length !== 4) + throw new Error('Expected all four shards'); + if (typeof plan.staticMutants !== 'boolean') throw new Error('Missing static mutation mode'); + const seen = new Set(); + const merged: MutationReport = { + schemaVersion: results.find(({ report }) => report)?.report?.schemaVersion ?? '1.0', + thresholds: { high: 90, low: 70 }, + files: {}, + testFiles: {}, + }; + const allKeys: string[] = []; + for (const { plan: other, completion, report } of results) { + const shard = plan.shards.find(({ index }) => index === completion.index); + if ( + JSON.stringify(other) !== JSON.stringify(plan) || + completion.digest !== plan.digest || + !completion.complete || + !shard || + seen.has(shard.index) || + completion.total !== shard.expected.length + ) { + throw new Error('Missing, duplicate, incomplete, or mismatched shard'); + } + seen.add(shard.index); + if (!report) { + if (shard.expected.length) throw new Error('Missing mutation report'); + continue; + } + if (report.schemaVersion !== merged.schemaVersion) throw new Error('Mismatched report schema'); + const prefix = (id: string) => `${shard.index}:${id}`; + const keys: string[] = []; + for (const [file, value] of Object.entries(report.files)) { + const existing = merged.files[file]; + if (existing && (existing.source !== value.source || existing.language !== value.language)) { + throw new Error(`Mismatched source: ${file}`); + } + const destination = existing ?? { ...value, mutants: [] }; + merged.files[file] = destination; + for (const mutant of value.mutants) { + if (!terminalStatuses.has(mutant.status)) throw new Error('Incomplete mutation result'); + if (mutant.replacement === undefined) throw new Error('Missing mutation replacement'); + keys.push(mutantKey(file, { ...mutant, replacement: mutant.replacement })); + destination.mutants.push({ + ...mutant, + id: prefix(mutant.id), + coveredBy: mutant.coveredBy?.map(prefix), + killedBy: mutant.killedBy?.map(prefix), + }); + } + } + assertMutants(shard.expected, keys); + allKeys.push(...keys); + for (const [file, value] of Object.entries(report.testFiles ?? {})) { + const existing = merged.testFiles![file]; + if (existing && existing.source !== value.source) + throw new Error(`Mismatched test source: ${file}`); + merged.testFiles![file] = { + ...value, + tests: [ + ...(existing?.tests ?? []), + ...value.tests.map((test) => ({ ...test, id: prefix(test.id) })), + ], + }; + } + } + assertMutants( + plan.shards.flatMap(({ expected }) => expected), + allKeys, + ); + return merged; +} + +export function mutationSummary(report: MutationReport, workspace: string, staticMutants = true) { + const counts = new Map(); + for (const file of Object.values(report.files)) + for (const mutant of file.mutants) { + counts.set(mutant.status, (counts.get(mutant.status) ?? 0) + 1); + } + const detected = (counts.get('Killed') ?? 0) + (counts.get('Timeout') ?? 0); + const tested = detected + (counts.get('Survived') ?? 0) + (counts.get('NoCoverage') ?? 0); + const score = tested ? `${((100 * detected) / tested).toFixed(2)}%` : 'N/A (no valid mutants)'; + return [ + `## ${workspace}: mutation testing (advisory)`, + '', + 'All four shards completed. Every planned mutation appears exactly once.', + staticMutants + ? 'Static=on. Static mutations are included.' + : 'Static=off. Static-only mutations are ignored. Hybrid mutations use per-test coverage.', + 'Ignored and invalid mutants are excluded from the score.', + '', + `Mutation score: ${score}. Calculated from combined counts, not averaged shard scores.`, + '', + '| Result | Mutants |', + '| --- | ---: |', + ...[...counts].sort().map(([status, count]) => `| ${status} | ${count} |`), + '', + 'The combined JSON report contains all mutations. Each shard artifact includes its HTML report and progress logs.', + '', + ].join('\n'); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const { values } = parseArgs({ + options: { + directory: { type: 'string' }, + workspace: { type: 'string' }, + head: { type: 'string' }, + prefix: { type: 'string', default: '' }, + }, + }); + if (!values.directory || !values.workspace || !values.head) + throw new Error('Expected --directory, --workspace and --head'); + const output = resolve('coverage/changes', values.workspace, 'mutation'); + rmSync(output, { recursive: true, force: true }); + mkdirSync(output, { recursive: true }); + try { + const results = [1, 2, 3, 4].map((index): ShardResult => { + const directory = resolve(values.directory!, `${values.prefix}shard-${index}`); + return { + plan: JSON.parse(readFileSync(`${directory}/plan.json`, 'utf8')), + completion: JSON.parse(readFileSync(`${directory}/completion.json`, 'utf8')), + report: existsSync(`${directory}/mutation.json`) + ? JSON.parse(readFileSync(`${directory}/mutation.json`, 'utf8')) + : undefined, + }; + }); + const report = mergeReports(results); + if (results[0].plan.workspace !== values.workspace || results[0].plan.head !== values.head) + throw new Error('Mismatched workspace or head'); + writeFileSync(`${output}/mutation.json`, JSON.stringify(report, null, 2)); + writeFileSync(`${output}/plan.json`, JSON.stringify(results[0].plan, null, 2)); + const summary = mutationSummary(report, values.workspace, results[0].plan.staticMutants); + writeFileSync(`${output}/summary.md`, summary); + console.log(summary); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } catch (error) { + const summary = `## ${values.workspace}: mutation testing incomplete\n\nA shard failed, is missing, or has mismatched results. No combined mutation score is available.\n`; + writeFileSync(`${output}/summary.md`, summary); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + throw error; + } +} diff --git a/scripts/quality/progress.mts b/scripts/quality/progress.mts index db3301392b..b5aa412896 100644 --- a/scripts/quality/progress.mts +++ b/scripts/quality/progress.mts @@ -24,6 +24,7 @@ export class FunctionProgressReporter implements Reporter { static inject = tokens(commonTokens.options); private readonly output: string; + private readonly staticMutants: boolean; private readonly functions: FunctionProgress[]; private readonly planned = new Map(); private readonly results = new Map>(); @@ -34,7 +35,10 @@ export class FunctionProgressReporter implements Reporter { private planReady = false; private complete = false; - constructor(options: Pick) { + constructor( + options: Pick & Partial>, + ) { + this.staticMutants = !options.ignoreStatic; this.output = dirname(options.jsonReporter.fileName); const targets: Target[] = JSON.parse(readFileSync(`${this.output}/functions.json`, 'utf8')); this.functions = targets.flatMap(({ file, functions }) => @@ -129,7 +133,9 @@ export class FunctionProgressReporter implements Reporter { } private log(message: string) { - console.log(`[mutation ${duration(Date.now() - this.started)}] ${message}`); + console.log( + `[mutation ${duration(Date.now() - this.started)} static=${this.staticMutants ? 'on' : 'off'}] ${message}`, + ); } private render() { @@ -148,6 +154,7 @@ export class FunctionProgressReporter implements Reporter { })); const snapshot = { phase: this.phase, + staticMutants: this.staticMutants, complete: this.complete, elapsedMs: Date.now() - this.started, lastResultAgeMs: this.lastResult === undefined ? null : Date.now() - this.lastResult, diff --git a/scripts/quality/quality.test.mts b/scripts/quality/quality.test.mts index 535ad17ca1..b0ce82f844 100644 --- a/scripts/quality/quality.test.mts +++ b/scripts/quality/quality.test.mts @@ -3,9 +3,15 @@ import { tmpdir } from 'node:os'; import { resolve, join } from 'node:path'; import { createRequire } from 'node:module'; import { strict as assert } from 'node:assert'; -import { spawnSync } from 'node:child_process'; import { Instrumenter } from '@stryker-mutator/instrumenter'; -import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { cognitiveFunctions, @@ -16,7 +22,7 @@ import { } from './risk.mts'; import { changedRanges, isSource, overlaps, workspaceFor, WORKSPACES } from './changes.mts'; import { measureFunctions } from '../crap.mts'; -import { coverageArgs } from './jest.mts'; +import { coverageGroups, runCoverage } from './jest.mts'; test('every application workspace has a source owner; test/support/declaration files are excluded', () => { for (const workspace of WORKSPACES) { @@ -212,7 +218,7 @@ function factory() { ); }); -test('the real Jest command collects every selected source file and excludes unrelated source', (t) => { +test('real Jest groups reuse compiled code, isolate Redis settings and preserve reports', (t) => { const directory = realpathSync(mkdtempSync(join(tmpdir(), 'quality-jest-'))); t.after(() => rmSync(directory, { recursive: true, force: true })); const require = createRequire(import.meta.url); @@ -220,30 +226,90 @@ test('the real Jest command collects every selected source file and excludes unr for (const file of [...files, 'untouched.js', 'secondp.js']) { writeFileSync(join(directory, file), 'module.exports = () => 1;'); } - writeFileSync(join(directory, 'jest.config.json'), JSON.stringify({ rootDir: directory })); + const transforms = join(directory, 'transforms.jsonl'); + writeFileSync( + join(directory, 'transform.cjs'), + `const fs = require('node:fs'); +module.exports = { process(code, filename, options) { + fs.appendFileSync(${JSON.stringify(transforms)}, JSON.stringify({ filename, instrument: options.instrument }) + '\\n'); + return { code }; +} };`, + ); + writeFileSync( + join(directory, 'jest.config.json'), + JSON.stringify({ + rootDir: directory, + maxWorkers: 2, + workerIdleMemoryLimit: '1GB', + cacheDirectory: join(directory, 'cache'), + transform: { '^.+\\.js$': join(directory, 'transform.cjs') }, + }), + ); writeFileSync( join(directory, 'sample.test.js'), - "test('calls sources', () => { expect(require('./first')()).toBe(1); expect(require('./second[part]')()).toBe(1); expect(require('./untouched')()).toBe(1); expect(require('./secondp')()).toBe(1); });", + "test('unit source', () => { expect(process.env.USE_REDIS).toBe('false'); expect(require('./first')()).toBe(1); expect(require('./untouched')()).toBe(1); expect(require('./secondp')()).toBe(1); });", + ); + writeFileSync( + join(directory, 'sample.cache_integration.spec.js'), + "test('Redis source', () => { expect(process.env.USE_REDIS).toBe('true'); expect(process.env.USE_REDIS_CLUSTER).toBe('false'); expect(require('./second[part]')()).toBe(1); });", ); const output = join(directory, 'coverage'); - const result = spawnSync( - process.execPath, - [ - require.resolve('jest/bin/jest'), - ...coverageArgs('jest.config.json', files, output, ['sample.test.js']), - ], - { - cwd: directory, - encoding: 'utf8', - env: { ...process.env, NODE_ENV: 'test' }, - }, - ); - assert.equal(result.status, 0, result.stderr); + const options = { + workspace: 'api', + config: 'jest.config.json', + cwd: directory, + jestBin: require.resolve('jest/bin/jest'), + files, + tests: ['sample.test.js', 'sample.cache_integration.spec.js'], + output, + env: { ...process.env, NODE_ENV: 'test', USE_REDIS: 'false' }, + }; + runCoverage(options); const coverage = JSON.parse(readFileSync(join(output, 'coverage-final.json'), 'utf8')); assert.deepEqual( Object.keys(coverage).sort(), files.map((file) => join(directory, file)), ); + for (const file of files) + assert.ok(Object.values(coverage[join(directory, file)].f).every((hits) => Number(hits) > 0)); + const compiled: { filename: string; instrument: boolean }[] = readFileSync(transforms, 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + for (const file of files) { + assert.equal( + compiled.filter((entry) => entry.filename === join(directory, file) && entry.instrument) + .length, + 1, + `${file} should be compiled once across coverage groups`, + ); + } + for (const group of ['unit', 'cache-core']) { + const results = JSON.parse(readFileSync(join(output, group, 'tests.json'), 'utf8')); + assert.equal(results.numPassedTests, 1); + assert.ok(existsSync(join(output, group, 'coverage-final.json'))); + } + writeFileSync( + join(directory, 'sample.cache_integration.spec.js'), + "test('failure is preserved', () => expect(1).toBe(2));", + ); + writeFileSync(join(directory, 'sample.stream_integration.spec.js'), "test('later', () => {});"); + const failedOutput = join(directory, 'failed'); + assert.throws( + () => + runCoverage({ + ...options, + output: failedOutput, + tests: [...options.tests, 'sample.stream_integration.spec.js'], + }), + /Jest cache-core exited with 1/u, + ); + const failed = JSON.parse(readFileSync(join(failedOutput, 'cache-core', 'tests.json'), 'utf8')); + assert.equal(failed.numFailedTests, 1); + assert.ok(existsSync(join(failedOutput, 'unit', 'coverage-final.json'))); + assert.ok(existsSync(join(failedOutput, 'cache-core', 'coverage-final.json'))); + assert.equal(existsSync(join(failedOutput, 'coverage-final.json')), false); + assert.equal(existsSync(join(failedOutput, 'cache-stream')), false); }); test('risk report keeps only changed functions, joins real coverage, and preserves missing coverage', (t) => { @@ -282,3 +348,34 @@ test('risk report keeps only changed functions, joins real coverage, and preserv assert.match(riskMarkdown(report), /N\/A/); assert.equal(riskReport(coverage, [{ file, ranges: [] }]).functions.length, 0); }); + +test('every related test belongs to one coverage group with the existing cache lane ordering', () => { + const tests = [ + '/repo/src/unit.spec.ts', + '/repo/src/cache/one.cache_integration.spec.ts', + '/repo/src/middleware/two.cache_integration.test.ts', + '/repo/src/cluster/three.cache_integration.spec.ts', + '/repo/src/mcp/four.cache_integration.spec.ts', + '/repo/src/stream/five.stream_integration.spec.ts', + ]; + const groups = coverageGroups('api', tests); + assert.deepEqual( + groups.flatMap((group) => group.tests), + tests, + ); + assert.deepEqual( + groups.map(({ name, redis, serial }) => ({ name, redis, serial })), + [ + { name: 'unit', redis: false, serial: false }, + { name: 'cache-core', redis: true, serial: false }, + { name: 'cache-cluster', redis: true, serial: true }, + { name: 'cache-mcp', redis: true, serial: false }, + { name: 'cache-stream', redis: true, serial: true }, + ], + ); + for (const workspace of ['server', 'data-schemas']) { + const [group] = coverageGroups(workspace, tests); + assert.deepEqual(group.tests, tests); + assert.equal(group.redis, false); + } +}); diff --git a/scripts/quality/queue.mts b/scripts/quality/queue.mts new file mode 100644 index 0000000000..7c6772fdb7 --- /dev/null +++ b/scripts/quality/queue.mts @@ -0,0 +1,230 @@ +import { resolve } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +import type { API, Comment, Context } from './comment.mts'; + +import { GATE, client, gateDecision, readRequest } from './comment.mts'; + +interface Group { + base_sha: string; + head_sha: string; + base_ref: string; + head_ref: string; +} +interface Comparison { + status: string; + total_commits: number; + merge_base_commit: { sha: string }; + commits: { sha: string; parents: { sha: string }[] }[]; +} +interface Pull { + number: number; + state: string; + headRefOid: string; + baseRefName: string; + comments: { + pageInfo: { hasNextPage: boolean }; + nodes: { + databaseId: number; + body: string; + author: { __typename: string; login: string } | null; + }[]; + }; +} +interface Association { + associatedPullRequests: { + pageInfo: { hasNextPage: boolean }; + nodes: Pull[]; + }; +} + +/** Merge parents preserve source heads. Squash/rebase groups cannot prove that identity. */ +export async function publishGroup(api: API, context: Context, group: Group) { + if ( + ![group.base_sha, group.head_sha].every((sha) => /^[a-f0-9]{40}$/u.test(sha)) || + group.base_sha === group.head_sha || + !group.base_ref.startsWith('refs/heads/') || + !group.head_ref.startsWith('refs/heads/gh-readonly-queue/') + ) + throw new Error('Invalid merge group'); + const status = (state: string, description: string) => + api.request('POST', `statuses/${group.head_sha}`, { + context: GATE, + state, + description, + target_url: `${context.server}/${context.repository}/actions/runs/${context.run}`, + }); + await status('pending', 'Verifying exact-head quality decisions for this merge group'); + try { + const path = `compare/${group.base_sha}...${group.head_sha}?per_page=100`; + const first = await api.request('GET', `${path}&page=1`); + if ( + first.status !== 'ahead' || + first.merge_base_commit.sha !== group.base_sha || + !Number.isSafeInteger(first.total_commits) || + first.total_commits < 1 || + first.total_commits > 1000 + ) + throw new Error('Unresolved merge group comparison'); + const rest = await Promise.all( + Array.from({ length: Math.ceil(first.total_commits / 100) - 1 }, (_, index) => + api.request('GET', `${path}&page=${index + 2}`), + ), + ); + const commits = [first, ...rest].flatMap((page) => page.commits); + const bySHA = new Map(commits.map((commit) => [commit.sha, commit])); + if (commits.length !== first.total_commits || bySHA.size !== commits.length) + throw new Error('Incomplete merge group comparison'); + const heads: string[] = []; + for (let sha = group.head_sha; sha !== group.base_sha; ) { + const commit = bySHA.get(sha); + if (!commit || commit.parents.length !== 2 || heads.length >= 100) + throw new Error('Unsupported merge group topology; merge commits are required'); + const head = commit.parents[1].sha; + if (!/^[a-f0-9]{40}$/u.test(head) || heads.includes(head)) + throw new Error('Ambiguous merge group source head'); + heads.push(head); + sha = commit.parents[0].sha; + } + const [owner, name] = context.repository.split('/'); + const pulls: Pull[] = []; + // Batch source-head associations rather than issuing a lookup for every commit. + for (let offset = 0; offset < heads.length; offset += 20) { + const batch = heads.slice(offset, offset + 20); + const result = await api.request<{ + errors?: object[]; + data?: { repository: Record }; + }>('POST', '/graphql', { + query: `query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + ${batch + .map( + (sha, index) => `p${index}: object(oid: "${sha}") { + ... on Commit { associatedPullRequests(first: 100) { + pageInfo { hasNextPage } + nodes { + number state headRefOid baseRefName + comments(first: 100) { + pageInfo { hasNextPage } + nodes { databaseId body author { __typename login } } + } + } + } } + }`, + ) + .join('\n')} + } + }`, + variables: { owner, name }, + }); + if (result.errors?.length || !result.data?.repository) + throw new Error('Merge group source associations unavailable'); + for (const [index, head] of batch.entries()) { + const associated = result.data.repository[`p${index}`]?.associatedPullRequests; + const matches = associated?.nodes.filter( + (pull) => + pull.state === 'OPEN' && + pull.headRefOid === head && + `refs/heads/${pull.baseRefName}` === group.base_ref, + ); + if (!associated || associated.pageInfo.hasNextPage || matches?.length !== 1) + throw new Error('Merge group source PR is missing or ambiguous'); + pulls.push(matches[0]); + } + } + const decisions = await Promise.all( + pulls.map(async (pull) => { + const items: Comment[] = pull.comments.nodes.map((item) => ({ + id: item.databaseId, + body: item.body, + user: { + type: item.author?.__typename ?? '', + login: + item.author?.__typename === 'Bot' + ? `${item.author.login}[bot]` + : (item.author?.login ?? ''), + }, + })); + for (let page = 2; pull.comments.pageInfo.hasNextPage; page++) { + const more = await api.request( + 'GET', + `issues/${pull.number}/comments?per_page=100&page=${page}`, + ); + items.push(...more); + if (more.length < 100) break; + } + const records = items.flatMap((item) => { + const request = readRequest(item); + return request?.pr === pull.number ? [request] : []; + }); + return gateDecision(records, pull.headRefOid); + }), + ); + const [ref, base] = await Promise.all( + [group.head_ref, group.base_ref].map((ref) => + api.request<{ object: { sha: string } }>('GET', `git/ref/${ref.replace(/^refs\//u, '')}`), + ), + ); + if (ref.object.sha !== group.head_sha || base.object.sha !== group.base_sha) + throw new Error('Merge group was superseded'); + const success = decisions.every((decision) => decision.state === 'success'); + await status( + success ? 'success' : 'failure', + success + ? 'Every incorporated PR head has an explicit successful quality decision' + : 'An incorporated PR head needs a successful quality request or explicit ignore', + ); + } catch (error) { + await status( + 'failure', + 'Cannot verify queue quality decisions; merge commits and exact PR heads are required', + ); + throw error; + } +} + +export async function publishQueueRun(api: API, context: Context, id: number) { + const run = await api.request<{ + event: string; + path: string; + head_sha: string; + head_branch: string; + status: string; + conclusion: string; + repository: { full_name: string }; + }>('GET', `actions/runs/${id}`); + if ( + run.event !== 'merge_group' || + run.path !== '.github/workflows/change-quality.yml' || + run.repository.full_name.toLowerCase() !== context.repository.toLowerCase() || + run.status !== 'completed' || + run.conclusion !== 'success' + ) + throw new Error('Expected a completed queue quality request'); + const branch = /^gh-readonly-queue\/(.+)\/pr-\d+-[a-f0-9]+$/u.exec(run.head_branch)?.[1]; + if (!branch) throw new Error('Invalid queue branch'); + const base = await api.request<{ object: { sha: string } }>('GET', `git/ref/heads/${branch}`); + await publishGroup(api, context, { + head_sha: run.head_sha, + head_ref: `refs/heads/${run.head_branch}`, + base_sha: base.object.sha, + base_ref: `refs/heads/${branch}`, + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const env = process.env; + const event = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH!, 'utf8')); + if (env.GITHUB_EVENT_NAME !== 'workflow_run') throw new Error('Expected a workflow completion'); + await publishQueueRun( + client(env), + { + repository: env.GITHUB_REPOSITORY!, + server: env.GITHUB_SERVER_URL!, + run: Number(env.GITHUB_RUN_ID), + attempt: Number(env.GITHUB_RUN_ATTEMPT), + }, + event.workflow_run.id, + ); +} diff --git a/scripts/quality/queue.test.mts b/scripts/quality/queue.test.mts new file mode 100644 index 0000000000..4ec2299780 --- /dev/null +++ b/scripts/quality/queue.test.mts @@ -0,0 +1,284 @@ +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; + +import type { API, Context, Request } from './comment.mts'; + +import { client, renderRequest } from './comment.mts'; +import { publishGroup, publishQueueRun } from './queue.mts'; + +const base = 'b'.repeat(40); +const source = 'a'.repeat(40); +const other = 'f'.repeat(40); +const intermediate = 'c'.repeat(40); +const head = 'd'.repeat(40); +const context: Context = { + repository: 'owner/repo', + server: 'https://github.com', + run: 55, + attempt: 1, +}; +const group = { + base_sha: base, + head_sha: head, + base_ref: 'refs/heads/dev', + head_ref: 'refs/heads/gh-readonly-queue/dev/pr-2-abc123', +}; +const request = (pr: number, sha: string): Request => ({ + request: pr + 10, + pr, + head: sha, + base, + run: pr + 50, + attempt: 1, + actor: 'maintainer', + phase: 'completed', + conclusion: 'success', + check: 'both', + scopes: ['data-schemas'], + staticMutants: false, +}); +const comment = (value: Request) => ({ + databaseId: 100 + value.pr, + body: renderRequest(value, context, value.head), + author: { __typename: 'Bot', login: 'github-actions' }, +}); +const pull = (pr: number, sha: string) => ({ + number: pr, + state: 'OPEN', + baseRefName: 'dev', + headRefOid: sha, + comments: { pageInfo: { hasNextPage: false }, nodes: [comment(request(pr, sha))] }, +}); + +class GitHub implements API { + calls: { method: string; path: string }[] = []; + statuses: { state: string; context: string; description: string }[] = []; + ref = head; + baseRef = base; + queueRun = { + event: 'merge_group', + path: '.github/workflows/change-quality.yml', + head_sha: head, + head_branch: group.head_ref.slice('refs/heads/'.length), + status: 'completed', + conclusion: 'success', + repository: { full_name: context.repository }, + }; + comparison = { + status: 'ahead', + total_commits: 4, + merge_base_commit: { sha: base }, + commits: [ + { sha: source, parents: [{ sha: base }] }, + { sha: intermediate, parents: [{ sha: base }, { sha: source }] }, + { sha: other, parents: [{ sha: base }] }, + { sha: head, parents: [{ sha: intermediate }, { sha: other }] }, + ], + }; + associations = [pull(2, other), pull(1, source)].map((item) => ({ + associatedPullRequests: { pageInfo: { hasNextPage: false }, nodes: [item] }, + })); + extraComments: ReturnType[] = []; + graphQLError = false; + async request(method: string, path: string, body?: object): Promise { + this.calls.push({ method, path }); + let result: object; + if (path.startsWith('statuses/')) { + assert.equal(path, `statuses/${head}`); + this.statuses.push(body as (typeof this.statuses)[number]); + result = {}; + } else if (path.startsWith('compare/')) { + const page = Number(new URLSearchParams(path.split('?')[1]).get('page')); + result = { + ...this.comparison, + commits: this.comparison.commits.slice((page - 1) * 100, page * 100), + }; + } else if (path === '/graphql') { + assert.deepEqual((body as { variables: object }).variables, { owner: 'owner', name: 'repo' }); + result = this.graphQLError + ? { errors: [{ message: 'Unavailable' }] } + : { + data: { + repository: Object.fromEntries( + this.associations.map((value, index) => [`p${index}`, value]), + ), + }, + }; + } else if (path.startsWith('issues/')) { + assert.equal(path, 'issues/1/comments?per_page=100&page=2'); + result = this.extraComments.map((item) => ({ + id: item.databaseId, + body: item.body, + user: { type: item.author.__typename, login: `${item.author.login}[bot]` }, + })); + } else if (path === 'actions/runs/55') { + result = this.queueRun; + } else if (path === 'git/ref/heads/dev') { + result = { object: { sha: this.baseRef } }; + } else if (path.startsWith('git/ref/')) { + assert.equal(path, 'git/ref/heads/gh-readonly-queue/dev/pr-2-abc123'); + result = { object: { sha: this.ref } }; + } else throw new Error(`Unexpected API call: ${method} ${path}`); + return structuredClone(result) as T; + } +} + +test('a merge group inherits successful exact-head decisions for every source PR in one batch', async () => { + const api = new GitHub(); + await publishGroup(api, context, group); + assert.deepEqual( + api.statuses.map(({ state }) => state), + ['pending', 'success'], + ); + assert.ok(api.statuses.every((status) => status.context === 'Quality decision')); + assert.equal(api.calls.filter(({ path }) => path === '/graphql').length, 1); + assert.equal(api.calls.filter(({ path }) => path.startsWith('issues/')).length, 0); +}); + +test('explicit ignore is accepted, but missing, failed, running, stale and spoofed decisions block', async () => { + for (const mode of ['ignore', 'missing', 'failure', 'running', 'stale', 'spoofed', 'other-pr']) { + const api = new GitHub(); + const item = api.associations[1].associatedPullRequests.nodes[0]; + const value = request(1, source); + if (mode === 'ignore') Object.assign(value, { check: 'ignore', scopes: [], phase: 'ignored' }); + if (mode === 'failure') value.conclusion = 'failure'; + if (mode === 'running') Object.assign(value, { phase: 'running', conclusion: null }); + if (mode === 'stale') value.head = base; + if (mode === 'other-pr') value.pr = 99; + item.comments.nodes = mode === 'missing' ? [] : [comment(value)]; + if (mode === 'spoofed') + item.comments.nodes[0].author = { __typename: 'User', login: 'github-actions' }; + await publishGroup(api, context, group); + assert.equal(api.statuses.at(-1)?.state, mode === 'ignore' ? 'success' : 'failure', mode); + } +}); + +test('a later accepted request for an incorporated head supersedes an earlier success', async () => { + const api = new GitHub(); + const newer = { ...request(1, source), request: 100, phase: 'queued', conclusion: null }; + api.associations[1].associatedPullRequests.nodes[0].comments.nodes.push(comment(newer)); + await publishGroup(api, context, group); + assert.equal(api.statuses.at(-1)?.state, 'failure'); +}); + +test('comparison and comment pagination preserve all merge members and decisions', async () => { + const api = new GitHub(); + api.comparison.commits.unshift( + ...Array.from({ length: 100 }, (_, index) => ({ + sha: index.toString(16).padStart(40, '0'), + parents: [{ sha: base }], + })), + ); + api.comparison.total_commits = api.comparison.commits.length; + const comments = api.associations[1].associatedPullRequests.nodes[0].comments; + comments.pageInfo.hasNextPage = true; + comments.nodes = Array.from({ length: 100 }, (_, index) => ({ + ...comment(request(1, source)), + body: 'ordinary comment', + databaseId: index, + })); + api.extraComments = [comment(request(1, source))]; + await publishGroup(api, context, group); + assert.equal(api.statuses.at(-1)?.state, 'success'); + assert.equal(api.calls.filter(({ path }) => path.startsWith('compare/')).length, 2); + assert.equal(api.calls.filter(({ path }) => path.startsWith('issues/')).length, 1); +}); + +test('unsupported, incomplete, ambiguous and superseded groups fail closed', async () => { + const cases: ((api: GitHub) => void)[] = [ + (api) => { + api.comparison.commits.at(-1)!.parents.pop(); + }, + (api) => { + api.comparison.commits = api.comparison.commits.filter(({ sha }) => sha !== intermediate); + api.comparison.total_commits--; + }, + (api) => { + api.comparison.total_commits++; + }, + (api) => { + api.comparison.total_commits = 1001; + }, + (api) => { + api.comparison.merge_base_commit.sha = source; + }, + (api) => { + api.comparison.status = 'diverged'; + }, + (api) => { + api.comparison.commits[0].sha = intermediate; + }, + (api) => { + api.associations[0].associatedPullRequests.nodes = []; + }, + (api) => { + api.associations[0].associatedPullRequests.nodes.push(pull(3, other)); + }, + (api) => { + api.associations[0].associatedPullRequests.pageInfo.hasNextPage = true; + }, + (api) => { + api.associations[0].associatedPullRequests.nodes[0].headRefOid = base; + }, + (api) => { + api.associations[0].associatedPullRequests.nodes[0].baseRefName = 'main'; + }, + (api) => { + api.associations[0].associatedPullRequests.nodes[0].state = 'CLOSED'; + }, + (api) => { + api.graphQLError = true; + }, + (api) => { + api.ref = source; + }, + (api) => { + api.baseRef = source; + }, + ]; + for (const change of cases) { + const api = new GitHub(); + change(api); + await assert.rejects(publishGroup(api, context, group)); + assert.equal(api.statuses.at(-1)?.state, 'failure'); + assert.ok(!api.statuses.some(({ state }) => state === 'success')); + } +}); + +test('trusted workflow completion resolves the queue and revalidates its protected base', async () => { + const api = new GitHub(); + await publishQueueRun(api, context, 55); + assert.equal(api.statuses.at(-1)?.state, 'success'); + assert.equal(api.calls.filter(({ path }) => path === 'git/ref/heads/dev').length, 2); +}); + +test('unrelated, failed, active and malformed workflow runs cannot publish queue decisions', async () => { + for (const overrides of [ + { event: 'pull_request' }, + { path: '.github/workflows/other.yml' }, + { repository: { full_name: 'other/repo' } }, + { status: 'in_progress' }, + { conclusion: 'failure' }, + { head_branch: 'untrusted-branch' }, + ]) { + const api = new GitHub(); + Object.assign(api.queueRun, overrides); + await assert.rejects(publishQueueRun(api, context, 55)); + assert.equal(api.statuses.length, 0); + } +}); + +test('the shared client routes GraphQL and REST to their distinct API paths', async (t) => { + const urls: string[] = []; + t.mock.method(globalThis, 'fetch', async (url: string) => { + urls.push(url); + return new Response('{}'); + }); + const api = client({ GITHUB_REPOSITORY: 'owner/repo' }); + await api.request('POST', '/graphql', { query: '{}' }); + await api.request('GET', 'pulls/1'); + assert.deepEqual(urls, [ + 'https://api.github.com/graphql', + 'https://api.github.com/repos/owner/repo/pulls/1', + ]); +}); diff --git a/scripts/quality/run.mts b/scripts/quality/run.mts index f83a9fa54f..32c610b420 100644 --- a/scripts/quality/run.mts +++ b/scripts/quality/run.mts @@ -3,19 +3,12 @@ import { createRequire } from 'node:module'; import { resolve, relative } from 'node:path'; import { spawnSync } from 'node:child_process'; import { Stryker } from '@stryker-mutator/core'; -import { - appendFileSync, - existsSync, - mkdirSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs'; +import { appendFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { mutationSelection, riskMarkdown, riskReport } from './risk.mts'; import { changes, git, WORKSPACES, workspaceFor } from './changes.mts'; -import { parseShard, shardTargets } from './shard.mts'; -import { coverageArgs } from './jest.mts'; +import { runCoverage } from './jest.mts'; +import { assertMutants, mutantKey, mutationPatterns, planShards } from './shards.mts'; const { values } = parseArgs({ options: { @@ -24,60 +17,47 @@ const { values } = parseArgs({ workspace: { type: 'string' }, check: { type: 'string' }, shard: { type: 'string' }, + static: { type: 'string', default: 'off' }, }, }); const workspace = WORKSPACES.find(({ name }) => name === values.workspace); if (!workspace || !['risk', 'mutation'].includes(values.check ?? '')) { throw new Error( - 'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation [--shard INDEX/COUNT]', + 'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation', ); } -const shard = parseShard(values.shard, values.check); -const shardLabel = shard ? ` (shard ${shard.index}/${shard.total})` : ''; +if (!['on', 'off'].includes(values.static)) throw new Error('--static must be on or off'); +const staticMutants = values.static === 'on'; +const staticDescription = staticMutants + ? 'Static mutations are included.' + : 'Static-only mutations are ignored and excluded from the score. Hybrid mutations use per-test coverage.'; const selection = changes(values.base, values.head); if (selection.head !== git(['rev-parse', 'HEAD']).trim()) { throw new Error('Check out the selected head before measuring its source'); } const files = selection.files.filter(({ file }) => workspaceFor(file)?.name === workspace.name); -const output = resolve('coverage/changes', workspace.name, values.check!); +const shardIndex = values.shard === undefined ? undefined : Number(values.shard); +if ( + shardIndex !== undefined && + (values.check !== 'mutation' || !Number.isInteger(shardIndex) || shardIndex < 1 || shardIndex > 4) +) { + throw new Error('--shard requires a mutation check and an index from 1 to 4'); +} +const output = resolve( + 'coverage/changes', + workspace.name, + shardIndex === undefined ? values.check! : `mutation-${shardIndex}`, +); rmSync(output, { recursive: true, force: true }); mkdirSync(output, { recursive: true }); -writeFileSync( - `${output}/selection.json`, - JSON.stringify( - { ...selection, workspace: workspace.name, files, ...(shard ? { shard } : {}) }, - null, - 2, - ), -); +writeFileSync(`${output}/selection.json`, JSON.stringify({ ...selection, files }, null, 2)); const summary = (markdown: string) => { console.log(markdown); writeFileSync(`${output}/summary.md`, markdown); if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); }; -const emptyMutationReports = () => { - writeFileSync( - `${output}/mutation.json`, - JSON.stringify({ schemaVersion: '1.0', files: {} }, null, 2), - ); - writeFileSync( - `${output}/progress.json`, - JSON.stringify( - { complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 }, - null, - 2, - ), - ); -}; -if (!files.length) { - if (shard) { - writeFileSync(`${output}/all-functions.json`, '[]'); - writeFileSync(`${output}/functions.json`, '[]'); - emptyMutationReports(); - } - summary( - `## ${workspace.name}: ${values.check}${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No changed source files.'}\n`, - ); +if (!files.length && shardIndex === undefined) { + summary(`## ${workspace.name}: ${values.check}\n\nNo changed source files.\n`); process.exit(0); } // Test binaries and configuration belong to the measured head, not the tooling checkout. @@ -110,25 +90,24 @@ if (values.check === 'risk') { throw new Error('Invalid Jest test selection'); } writeFileSync(`${output}/selected-tests.json`, JSON.stringify(tests, null, 2)); - const result = tests.length - ? spawnSync( - process.execPath, - [jestBin, ...coverageArgs(workspace.config, localFiles, output, tests)], - { cwd: workspace.path, stdio: 'inherit', env }, - ) - : { error: undefined, status: 0, signal: null }; - if (result.error || result.status !== 0) { + try { + runCoverage({ + workspace: workspace.name, + config: workspace.config, + cwd: workspace.path, + jestBin, + files: localFiles, + tests, + output, + env, + }); + } catch (error) { summary( `## ${workspace.name}: change risk\n\nCoverage failed. Scores are unavailable. See the test log.\n`, ); - throw result.error ?? new Error(`Jest exited with ${result.status ?? result.signal}`); + throw error; } const coverageFile = `${output}/coverage-final.json`; - if (!existsSync(coverageFile)) { - // Jest produces no coverage file when no test matches. Keep the lack of evidence explicit. - if (tests.length !== 0) throw new Error('Jest did not produce coverage'); - writeFileSync(coverageFile, '{}'); - } const report = riskReport(coverageFile, files, selection.base); writeFileSync( `${output}/risk.json`, @@ -138,16 +117,48 @@ if (values.check === 'risk') { } else { process.env.NODE_ENV = env.NODE_ENV; process.env.BABEL_ENV = env.BABEL_ENV; + console.log(`Mutation run: static=${values.static}. ${staticDescription}`); const allTargets = mutationSelection(files, selection.base); - const targets = shardTargets(allTargets, shard); - if (shard) { - writeFileSync(`${output}/all-functions.json`, JSON.stringify(allTargets, null, 2)); + const plan = + shardIndex === undefined + ? undefined + : await planShards( + allTargets, + selection.base, + selection.head, + workspace.name, + 4, + staticMutants, + ); + const shard = plan?.shards.find(({ index }) => index === shardIndex); + const targets = shard?.targets ?? allTargets; + if (plan && shard) { + writeFileSync(`${output}/plan.json`, JSON.stringify(plan, null, 2)); + console.log( + `Mutation shard ${shard.index}/${plan.shards.length}: ${shard.expected.length} planned mutants; two workers; static=${values.static}`, + ); } writeFileSync(`${output}/functions.json`, JSON.stringify(targets, null, 2)); - if (!targets.length) { - if (shard) emptyMutationReports(); + const completeShard = () => { + if (plan && shard) + writeFileSync( + `${output}/completion.json`, + JSON.stringify( + { + digest: plan.digest, + index: shard.index, + complete: true, + total: shard.expected.length, + }, + null, + 2, + ), + ); + }; + if (!targets.length || shard?.expected.length === 0) { + completeShard(); summary( - `## ${workspace.name}: mutation testing${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No added or edited functions.'}\n`, + `## ${workspace.name}: mutation testing\n\nNo mutations selected for this run. Static=${values.static}.\n`, ); process.exit(0); } @@ -175,8 +186,8 @@ if (values.check === 'risk') { : {}), }, }, - mutate: targets.flatMap(({ mutate }) => mutate), - ignoreStatic: false, + mutate: mutationPatterns(targets), + ignoreStatic: !staticMutants, coverageAnalysis: 'perTest', concurrency: 2, incremental: false, @@ -203,6 +214,13 @@ if (values.check === 'risk') { }); try { const results = await stryker.runMutationTest(); + if (shard) + assertMutants( + shard.expected, + results.map((mutant) => + mutantKey(relative(process.cwd(), mutant.fileName).replaceAll('\\', '/'), mutant), + ), + ); const progress = JSON.parse(readFileSync(`${output}/progress.json`, 'utf8')); if (!progress.complete || progress.completed !== results.length) { throw new Error('Mutation progress report is incomplete'); @@ -215,11 +233,11 @@ if (values.check === 'risk') { const score = tested ? `${((100 * detected) / tested).toFixed(2)}%` : 'N/A (no valid mutants)'; summary( [ - `## ${workspace.name}: mutation testing (advisory)${shardLabel}`, + `## ${workspace.name}: mutation testing (advisory)`, '', - `Changed functions: ${targets.reduce((total, target) => total + target.functions.length, 0)} in ${targets.length} files. Static mutations are included. Duration: ${((Date.now() - started) / 60000).toFixed(1)} minutes.`, + `Changed functions: ${targets.reduce((total, target) => total + target.functions.length, 0)} in ${targets.length} files. Static=${values.static}. ${staticDescription} Duration: ${((Date.now() - started) / 60000).toFixed(1)} minutes.`, '', - `Mutation score: ${score}. Surviving and uncovered mutants reduce this score. Invalid mutants are excluded.`, + `Mutation score: ${score}. Surviving and uncovered mutants reduce this score. Ignored and invalid mutants are excluded.`, ...(runtimeErrors ? [ `${runtimeErrors} mutations had runtime errors. The check fails because their results are incomplete.`, @@ -235,9 +253,10 @@ if (values.check === 'risk') { ].join('\n'), ); if (runtimeErrors) process.exitCode = 1; + else completeShard(); } catch (error) { summary( - `## ${workspace.name}: mutation testing${shardLabel}\n\nStryker failed. This is not a mutation score. See the test log.\n`, + `## ${workspace.name}: mutation testing\n\nStryker failed. This is not a mutation score. See the test log.\n`, ); throw error; } diff --git a/scripts/quality/shard.mts b/scripts/quality/shard.mts deleted file mode 100644 index 564d4a6f9c..0000000000 --- a/scripts/quality/shard.mts +++ /dev/null @@ -1,30 +0,0 @@ -export interface Shard { - index: number; - total: number; -} - -export function parseShard( - value: string | undefined, - check: string | undefined, -): Shard | undefined { - if (value === undefined) return undefined; - if (check !== 'mutation') throw new Error('--shard is only allowed for mutation checks'); - if (!/^[1-9]\d*\/[1-9]\d*$/u.test(value)) { - throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT'); - } - const [index, total] = value.split('/').map(Number); - if (!Number.isSafeInteger(index) || !Number.isSafeInteger(total) || index > total) { - throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT'); - } - return { index, total }; -} - -export function shardTargets( - targets: readonly T[], - shard?: Shard, -): readonly T[] { - if (!shard) return targets; - return [...targets] - .sort((a, b) => b.functions.length - a.functions.length || a.file.localeCompare(b.file)) - .filter((_target, index) => index % shard.total === shard.index - 1); -} diff --git a/scripts/quality/shard.test.mts b/scripts/quality/shard.test.mts deleted file mode 100644 index b58e313733..0000000000 --- a/scripts/quality/shard.test.mts +++ /dev/null @@ -1,234 +0,0 @@ -import { test } from 'node:test'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { strict as assert } from 'node:assert'; -import { spawnSync } from 'node:child_process'; -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; - -import { parseShard, shardTargets } from './shard.mts'; -import { mutationTarget } from './risk.mts'; -import { git } from './changes.mts'; - -const root = fileURLToPath(new URL('../../', import.meta.url)); -const runner = fileURLToPath(new URL('./run.mts', import.meta.url)); - -const targets = Object.freeze( - Array.from({ length: 8 }, (_value, index) => - Object.freeze( - mutationTarget( - `file-${index}.ts`, - 'const first = (ok) => ok ? true : false;\nconst second = () => false;', - [{ start: 1, count: 2 }], - ), - ), - ), -); - -for (const [index, expected] of [ - [1, [0, 3, 6]], - [2, [1, 4, 7]], - [3, [2, 5]], -] as const) { - test(`shard ${index}/3 retains whole files and their full function and mutation ranges`, () => { - const shard = parseShard(`${index}/3`, 'mutation'); - const chosen = shardTargets(targets, shard); - assert.deepEqual( - chosen, - expected.map((offset) => targets[offset]), - ); - for (const [position, offset] of expected.entries()) { - assert.equal(chosen[position], targets[offset]); - assert.equal(chosen[position].functions.length, 2); - assert.equal(chosen[position].mutate, targets[offset].mutate); - } - assert.deepEqual(shardTargets(targets, shard), chosen); - }); -} - -test('three shards contain every full target exactly once without mutating input order', () => { - const before = JSON.stringify(targets); - const union = [1, 2, 3].flatMap((index) => shardTargets(targets, { index, total: 3 })); - assert.equal(union.length, targets.length); - assert.equal(new Set(union).size, targets.length); - assert.deepEqual(new Set(union), new Set(targets)); - assert.deepEqual( - union.flatMap(({ mutate }) => mutate).sort(), - targets.flatMap(({ mutate }) => mutate).sort(), - ); - assert.equal(JSON.stringify(targets), before); -}); - -test('omitting --shard retains all original targets and allows risk checks', () => { - assert.equal(parseShard(undefined, 'mutation'), undefined); - assert.equal(parseShard(undefined, 'risk'), undefined); - assert.equal(shardTargets(targets), targets); - assert.deepEqual(shardTargets(targets, parseShard('1/1', 'mutation')), targets); -}); - -test('equal function counts use file order regardless of input order without mutating input', () => { - const reversed = Object.freeze([...targets].reverse()); - const before = JSON.stringify(reversed); - for (const index of [1, 2, 3]) { - const shard = parseShard(`${index}/3`, 'mutation'); - assert.deepEqual(shardTargets(reversed, shard), shardTargets(targets, shard)); - } - assert.equal(JSON.stringify(reversed), before); - assert.equal(shardTargets(reversed), reversed); - assert.deepEqual(shardTargets(reversed, parseShard('1/1', 'mutation')), targets); -}); - -test('function-weighted round robin spreads the known target file workload across three shards', () => { - const workload = Object.freeze( - ( - [ - ['pool.ts', 25], - ['import.ts', 19], - ['nativefavorite.ts', 13], - ['mongo.ts', 11], - ['migrations.ts', 9], - ['user.ts', 8], - ['backend.ts', 3], - ['favorites/mongo.ts', 1], - ['methods/createMethods.ts', 1], - ] as const - ).map(([file, count]) => - Object.freeze({ - file, - functions: Object.freeze(Array.from({ length: count }, (_value, index) => index)), - }), - ), - ); - const before = JSON.stringify(workload); - const expected = [ - ['pool.ts', 'mongo.ts', 'backend.ts'], - ['import.ts', 'migrations.ts', 'favorites/mongo.ts'], - ['nativefavorite.ts', 'user.ts', 'methods/createMethods.ts'], - ]; - const shuffled = Object.freeze([...workload].reverse()); - const shards = [1, 2, 3].map((index) => { - const shard = parseShard(`${index}/3`, 'mutation'); - const chosen = shardTargets(shuffled, shard); - assert.deepEqual( - chosen.map(({ file }) => file), - expected[index - 1], - ); - assert.deepEqual(chosen, shardTargets(workload, shard)); - return chosen; - }); - const union = shards.flat(); - assert.equal(union.length, workload.length); - assert.equal(new Set(union).size, workload.length); - assert.deepEqual(new Set(union), new Set(workload)); - assert.deepEqual( - shards.map((chosen) => chosen.reduce((sum, target) => sum + target.functions.length, 0)), - [39, 29, 22], - ); - assert.equal(JSON.stringify(workload), before); - assert.equal(shardTargets(shuffled), shuffled); -}); - -test('a shard beyond the number of selected target files is empty', () => { - assert.deepEqual(shardTargets(targets.slice(0, 1), parseShard('2/3', 'mutation')), []); - assert.deepEqual(shardTargets([], parseShard('1/3', 'mutation')), []); -}); - -for (const value of [ - '0/2', - '4/3', - 'foo', - '1/0', - '1.1/2', - '1/2/3', - '', - '-1/2', - '1/-2', - '01/2', - '1/02', - '+1/2', - ' 1/2', - '1/2 ', - '1e1/20', - '1/9007199254740992', -]) { - test(`invalid shard ${JSON.stringify(value)} is rejected`, () => { - assert.throws(() => parseShard(value, 'mutation'), /--shard must be INDEX\/COUNT/u); - }); -} - -test('risk checks reject --shard before attempting source selection', () => { - assert.throws(() => parseShard('1/3', 'risk'), /only allowed for mutation/u); - const result = spawnSync( - process.execPath, - [ - runner, - '--workspace', - 'data-schemas', - '--check', - 'risk', - '--shard', - '1/3', - '--base', - 'nonexistent-shard-test-ref', - ], - { cwd: root, encoding: 'utf8', timeout: 15000 }, - ); - assert.notEqual(result.status, 0); - assert.match(result.stderr, /--shard is only allowed for mutation checks/u); - assert.doesNotMatch(result.stderr, /Not a valid object name/u); -}); - -test('empty sharded CLI runs pass with truthful metadata; unsharded output stays unchanged', (t) => { - const directory = mkdtempSync(join(tmpdir(), 'quality-shard-')); - t.after(() => rmSync(directory, { recursive: true, force: true })); - const env = { - ...process.env, - GIT_DIR: git(['rev-parse', '--absolute-git-dir'], root).trim(), - GIT_WORK_TREE: directory, - GITHUB_STEP_SUMMARY: '', - }; - const output = join(directory, 'coverage/changes/data-schemas/mutation'); - for (const shard of ['2/3', undefined]) { - const result = spawnSync( - process.execPath, - [ - runner, - '--workspace', - 'data-schemas', - '--check', - 'mutation', - '--base', - 'HEAD', - ...(shard ? ['--shard', shard] : []), - ], - { cwd: directory, env, encoding: 'utf8', timeout: 15000 }, - ); - assert.equal(result.status, 0, `${result.error ?? ''}\n${result.stdout}\n${result.stderr}`); - const selection = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8')); - assert.equal(selection.workspace, 'data-schemas'); - assert.deepEqual(selection.files, []); - if (shard) { - assert.deepEqual(selection.shard, { index: 2, total: 3 }); - assert.deepEqual(JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')), { - schemaVersion: '1.0', - files: {}, - }); - assert.deepEqual(JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')), { - complete: true, - completed: 0, - total: 0, - staticCompleted: 0, - staticTotal: 0, - }); - assert.deepEqual(JSON.parse(readFileSync(join(output, 'all-functions.json'), 'utf8')), []); - assert.deepEqual(JSON.parse(readFileSync(join(output, 'functions.json'), 'utf8')), []); - assert.match(result.stdout, /mutation \(shard 2\/3\)/u); - assert.match(result.stdout, /No changed functions for shard\./u); - } else { - assert.equal(selection.shard, undefined); - assert.equal(existsSync(join(output, 'all-functions.json')), false); - assert.equal(existsSync(join(output, 'functions.json')), false); - assert.equal(result.stdout, '## data-schemas: mutation\n\nNo changed source files.\n\n'); - } - } -}); diff --git a/scripts/quality/shards.mts b/scripts/quality/shards.mts new file mode 100644 index 0000000000..3213addefd --- /dev/null +++ b/scripts/quality/shards.mts @@ -0,0 +1,173 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { relative, resolve } from 'node:path'; +import { Instrumenter } from '@stryker-mutator/instrumenter'; + +import type { Mutant, MutationRange } from '@stryker-mutator/api/core'; +import type { mutationSelection } from './risk.mts'; + +type Target = ReturnType[number]; + +/** Stryker normalizes backslashes as path separators, so use character classes for literals. */ +export function mutationPatterns(targets: Target[]): string[] { + return targets.flatMap(({ file, mutate }) => + mutate.map((pattern) => { + const suffix = /:\d+:\d+-\d+:\d+$/u.exec(pattern)?.[0]; + if (!suffix || file.includes('\\')) throw new Error(`Unsupported mutation path: ${file}`); + return file.replace(/[*?[\]{}()!+@]/gu, (character) => `[${character}]`) + suffix; + }), + ); +} +export interface Shard { + index: number; + targets: Target[]; + expected: string[]; +} +export interface ShardPlan { + base: string; + head: string; + workspace: string; + digest: string; + staticMutants: boolean; + shards: Shard[]; +} + +const compare = (a: { line: number; column: number }, b: typeof a) => + a.line - b.line || a.column - b.column; + +export function mutationRange(pattern: string): MutationRange { + const match = /:(\d+):(\d+)-(\d+):(\d+)$/u.exec(pattern); + if (!match) throw new Error(`Invalid mutation range: ${pattern}`); + return { + start: { line: Number(match[1]) - 1, column: Number(match[2]) }, + end: { line: Number(match[3]) - 1, column: Number(match[4]) }, + }; +} + +/** IDs change between Stryker runs. Source locations and replacements identify the mutation. */ +export function mutantKey( + file: string, + mutant: Pick, +) { + const { start, end } = mutant.location; + return JSON.stringify([ + file, + start.line, + start.column, + end.line, + end.column, + mutant.mutatorName, + mutant.replacement, + ]); +} + +export function assertMutants(expected: string[], actual: string[]) { + const unique = new Set(actual); + if ( + new Set(expected).size !== expected.length || + unique.size !== actual.length || + actual.length !== expected.length || + expected.some((key) => !unique.has(key)) + ) { + throw new Error( + 'Mutation set differs from the shard plan (missing, duplicate, or unexpected mutant)', + ); + } +} + +export async function planShards( + targets: Target[], + base: string, + head: string, + workspace: string, + count = 4, + staticMutants = true, +): Promise { + if (!Number.isInteger(count) || count < 1 || count > 4) throw new Error('Expected 1 to 4 shards'); + const sources = targets.map((target) => ({ + name: resolve(target.file), + content: readFileSync(target.file, 'utf8'), + mutate: target.mutate.map(mutationRange), + })); + const disabled = () => false; + const silent = () => {}; + const instrumenter = new Instrumenter({ + isTraceEnabled: disabled, + isDebugEnabled: disabled, + isInfoEnabled: disabled, + isWarnEnabled: disabled, + isErrorEnabled: disabled, + isFatalEnabled: disabled, + trace: silent, + debug: silent, + info: silent, + warn: silent, + error: silent, + fatal: silent, + }); + const { mutants } = await instrumenter.instrument(sources, { + plugins: null, + ignorers: [], + excludedMutations: [], + }); + const groups = targets.flatMap((target) => + target.mutate.map((pattern) => ({ + file: target.file, + pattern, + range: mutationRange(pattern), + expected: [] as string[], + })), + ); + for (const mutant of mutants) { + const file = relative(process.cwd(), mutant.fileName).replaceAll('\\', '/'); + const owners = groups.filter( + (group) => + group.file === file && + compare(group.range.start, mutant.location.start) <= 0 && + compare(mutant.location.end, group.range.end) <= 0, + ); + if (owners.length !== 1) throw new Error(`Mutation must belong to exactly one range: ${file}`); + // Instrumenter positions are zero-based; JSON report positions are one-based. + const location = { + start: { line: mutant.location.start.line + 1, column: mutant.location.start.column + 1 }, + end: { line: mutant.location.end.line + 1, column: mutant.location.end.column + 1 }, + }; + owners[0].expected.push(mutantKey(file, { ...mutant, location })); + } + const shards: Shard[] = Array.from({ length: count }, (_, index) => ({ + index: index + 1, + targets: [], + expected: [], + })); + // Preserve the existing disjoint ranges. Cutting an outer block would lose its mutant. + for (const group of groups.sort( + (a, b) => b.expected.length - a.expected.length || a.pattern.localeCompare(b.pattern, 'en'), + )) { + const shard = [...shards].sort( + (a, b) => a.expected.length - b.expected.length || a.index - b.index, + )[0]; + let target = shard.targets.find((item) => item.file === group.file); + if (!target) { + target = { ...targets.find((item) => item.file === group.file)!, mutate: [] }; + shard.targets.push(target); + } + target.mutate.push(group.pattern); + shard.expected.push(...group.expected); + } + for (const shard of shards) shard.expected.sort(); + const digest = createHash('sha256') + .update( + JSON.stringify({ + base, + head, + workspace, + staticMutants, + targets, + shards, + sources: sources.map(({ content }) => content), + lock: readFileSync('package-lock.json', 'utf8'), + }), + ) + .digest('hex'); + return { base, head, workspace, digest, staticMutants, shards }; +} diff --git a/scripts/quality/shards.test.mts b/scripts/quality/shards.test.mts new file mode 100644 index 0000000000..7422f9d4f2 --- /dev/null +++ b/scripts/quality/shards.test.mts @@ -0,0 +1,243 @@ +import { test } from 'node:test'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { strict as assert } from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; + +import type { ShardResult, MutationReport } from './merge.mts'; +import type { ShardPlan } from './shards.mts'; + +import { mergeReports, mutationSummary } from './merge.mts'; +import { assertMutants, mutantKey } from './shards.mts'; + +test('four real Stryker runs preserve the unsharded mutant set, including nested and static code', () => { + const directory = realpathSync(mkdtempSync(join(tmpdir(), 'quality-shards-'))); + const source = `function factory() { + function positive(value) { return value > 0; } + return { positive, enabled: true }; +} +function negative(value) { return value < 0; } +function zero(value) { return value === 0; } +function untouched() { return false; } +module.exports = { ...factory(), negative, zero, untouched };\n`; + writeFileSync(join(directory, 'subject[one].js'), source); + writeFileSync(join(directory, 'package-lock.json'), '{}'); + writeFileSync( + join(directory, 'test.cjs'), + `const assert = require('node:assert/strict'); +const subject = require('./subject[one].js'); +assert.equal(subject.positive(1), true); +assert.equal(subject.positive(0), false); +assert.equal(subject.negative(-1), true); +assert.equal(subject.zero(0), true); +assert.equal(subject.enabled, true);\n`, + ); + const script = `import { Stryker } from ${JSON.stringify(import.meta.resolve('@stryker-mutator/core'))}; +import { mutationTarget } from ${JSON.stringify(new URL('./risk.mts', import.meta.url).href)}; +import { planShards, mutantKey, assertMutants, mutationPatterns } from ${JSON.stringify(new URL('./shards.mts', import.meta.url).href)}; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +const source = readFileSync('subject[one].js', 'utf8'); +const target = mutationTarget('subject[one].js', source, [{start: 1, count: 6}], source.replace('> 0', '> 1').replace('enabled: true', 'enabled: false').replace('< 0', '< 1').replace('=== 0', '=== 1')); +const plan = await planShards([target], 'base', 'head', 'fixture'); +writeFileSync('plan.json', JSON.stringify(plan)); +for (const index of [0,1,2,3,4]) { + const shard = plan.shards[index - 1]; + const targets = shard?.targets ?? [target]; + const output = 'report-' + index; + mkdirSync(output); + writeFileSync(output + '/functions.json', JSON.stringify(targets)); + if (shard?.expected.length === 0) continue; + const results = await new Stryker({ + plugins: [${JSON.stringify(new URL('./progress.mts', import.meta.url).href)}], + testRunner: 'command', commandRunner: { command: ${JSON.stringify(JSON.stringify(process.execPath) + ' test.cjs')} }, + ignorePatterns: ['**', '!subject[[]one].js', '!test.cjs'], + mutate: mutationPatterns(targets), reporters: ['function-progress', 'json'], + jsonReporter: {fileName: output + '/mutation.json'}, + coverageAnalysis: 'off', concurrency: 1, timeoutMS: 1000, thresholds: {break: 0}, + }).runMutationTest(); + if (shard) assertMutants(shard.expected, results.map(m => mutantKey('subject[one].js', m))); +}`; + writeFileSync(join(directory, 'run.mjs'), script); + try { + const run = spawnSync(process.execPath, ['run.mjs'], { + cwd: directory, + encoding: 'utf8', + timeout: 120000, + }); + assert.equal(run.status, 0, run.stdout + run.stderr); + const plan: ShardPlan = JSON.parse(readFileSync(join(directory, 'plan.json'), 'utf8')); + const baseline: MutationReport = JSON.parse( + readFileSync(join(directory, 'report-0/mutation.json'), 'utf8'), + ); + const results: ShardResult[] = plan.shards.map((shard) => ({ + plan, + completion: { + digest: plan.digest, + index: shard.index, + complete: true, + total: shard.expected.length, + }, + report: shard.expected.length + ? JSON.parse(readFileSync(join(directory, `report-${shard.index}/mutation.json`), 'utf8')) + : undefined, + })); + const merged = mergeReports(results); + const keys = (report: MutationReport) => + Object.entries(report.files).flatMap(([file, value]) => + value.mutants.map((mutant) => + mutantKey(file, { ...mutant, replacement: mutant.replacement! }), + ), + ); + assertMutants(keys(baseline), keys(merged)); + assert.ok(keys(merged).length > 5); + assert.ok(plan.shards.filter((shard) => shard.expected.length > 0).length >= 3); + assert.ok( + Object.values(merged.files) + .flatMap((file) => file.mutants) + .every((mutant) => mutant.location.end.line <= 6), + ); + for (const shard of plan.shards.filter((item) => item.expected.length)) { + const progress = JSON.parse( + readFileSync(join(directory, `report-${shard.index}/progress.json`), 'utf8'), + ); + assert.equal(progress.complete, true); + assert.equal(progress.total, shard.expected.length); + } + assert.throws(() => mergeReports(results.slice(1)), /four shards/); + const changed = structuredClone(results); + changed[0].plan = { ...changed[0].plan, head: 'wrong-head' }; + assert.throws(() => mergeReports(changed), /mismatched shard/); + const mixedModes = structuredClone(results); + mixedModes[0].plan = { ...mixedModes[0].plan, staticMutants: false }; + assert.throws(() => mergeReports(mixedModes), /mismatched shard/); + const missing = structuredClone(results); + missing[0].completion.complete = false; + assert.throws(() => mergeReports(missing), /incomplete/); + assert.throws( + () => mergeReports([results[0], results[0], results[2], results[3]]), + /duplicate/, + ); + const incomplete = structuredClone(results); + Object.values(incomplete[0].report!.files)[0].mutants.pop(); + assert.throws(() => mergeReports(incomplete), /Mutation set differs/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('combined scores use mutant counts and reject duplicates', () => { + const result = (id: string, status: 'Killed' | 'Survived') => ({ + id, + status, + mutatorName: 'BooleanLiteral', + replacement: 'false', + location: { start: { line: 1, column: 1 }, end: { line: 1, column: 5 } }, + }); + const report: MutationReport = { + schemaVersion: '2', + thresholds: { high: 90, low: 70 }, + files: { + 'a.js': { source: 'true', language: 'javascript', mutants: [result('1', 'Killed')] }, + 'b.js': { + source: 'true', + language: 'javascript', + mutants: Array.from({ length: 9 }, (_, i) => result(String(i + 2), 'Survived')), + }, + }, + }; + assert.match(mutationSummary(report, 'fixture'), /10.00%/); + report.files['a.js'].mutants.push({ + ...result('ignored', 'Killed'), + status: 'Ignored', + static: true, + }); + assert.match(mutationSummary(report, 'fixture', false), /10.00%/); + assert.match(mutationSummary(report, 'fixture', false), /Static=off/); + assert.match(mutationSummary(report, 'fixture', false), /Ignored \| 1/); + assert.throws(() => assertMutants(['a', 'b'], ['a', 'a']), /duplicate/); + assert.throws(() => assertMutants(['a'], ['b']), /unexpected/); +}); + +test('merging namespaces test references and refuses unfinished or conflicting source reports', () => { + const reports: MutationReport[] = [1, 2, 3, 4].map((line) => ({ + schemaVersion: '1.0', + thresholds: { high: 90, low: 70 }, + files: { + 'shared.js': { + source: 'shared source', + language: 'javascript', + mutants: [ + { + id: '0', + status: 'Killed', + replacement: 'false', + mutatorName: 'BooleanLiteral', + location: { start: { line, column: 1 }, end: { line, column: 5 } }, + coveredBy: ['0'], + killedBy: ['0'], + }, + ], + }, + }, + testFiles: { 'test.js': { source: 'shared test', tests: [{ id: '0', name: 'test' }] } }, + })); + const plan: ShardPlan = { + base: 'base', + head: 'head', + workspace: 'fixture', + digest: 'digest', + staticMutants: true, + shards: reports.map((report, index) => ({ + index: index + 1, + targets: [], + expected: [ + mutantKey('shared.js', { ...report.files['shared.js'].mutants[0], replacement: 'false' }), + ], + })), + }; + const results = reports.map( + (report, index): ShardResult => ({ + plan, + report, + completion: { digest: plan.digest, index: index + 1, complete: true, total: 1 }, + }), + ); + const merged = mergeReports(results); + assert.deepEqual( + merged.testFiles!['test.js'].tests.map(({ id }) => id), + ['1:0', '2:0', '3:0', '4:0'], + ); + assert.deepEqual( + merged.files['shared.js'].mutants.map(({ killedBy }) => killedBy), + [['1:0'], ['2:0'], ['3:0'], ['4:0']], + ); + assert.equal(reports[0].files['shared.js'].mutants[0].id, '0'); + for (const status of ['Pending', 'RuntimeError', 'Unknown', undefined, null, 0, {}]) { + const unfinished = structuredClone(results); + Reflect.set(unfinished[0].report!.files['shared.js'].mutants[0], 'status', status); + assert.throws(() => mergeReports(unfinished), /Incomplete/); + } + const absent = structuredClone(results); + Reflect.deleteProperty(absent[0].report!.files['shared.js'].mutants[0], 'status'); + assert.throws(() => mergeReports(absent), /Incomplete/); + for (const status of ['Killed', 'Survived', 'Timeout', 'NoCoverage', 'CompileError', 'Ignored']) { + const complete = structuredClone(results); + Reflect.set(complete[0].report!.files['shared.js'].mutants[0], 'status', status); + assert.equal(mergeReports(complete).files['shared.js'].mutants[0].status, status); + } + const changed = structuredClone(results); + changed[1].report!.files['shared.js'].source = 'other source'; + assert.throws(() => mergeReports(changed), /Mismatched source/); + const emptyPlan: ShardPlan = { + ...plan, + shards: plan.shards.map((s) => ({ ...s, expected: [] })), + }; + const empty = mergeReports( + emptyPlan.shards.map((s) => ({ + plan: emptyPlan, + completion: { digest: plan.digest, index: s.index, complete: true, total: 0 }, + })), + ); + assert.match(mutationSummary(empty, 'fixture'), /N\/A/); +}); diff --git a/scripts/quality/workflow.test.mts b/scripts/quality/workflow.test.mts index b20d6173bf..8d951ec204 100644 --- a/scripts/quality/workflow.test.mts +++ b/scripts/quality/workflow.test.mts @@ -17,31 +17,49 @@ import { writeFileSync, } from 'node:fs'; -import { git } from './changes.mts'; +import type { MutationReport } from './merge.mts'; + +import { git, reportJobs } from './changes.mts'; +import { assertMutants, mutantKey } from './shards.mts'; +import { mutationSummary } from './merge.mts'; interface Step { + if?: string; id?: string; name?: string; uses?: string; run?: string; with?: Record; - 'working-directory'?: string; - if?: string; env?: Record; - 'continue-on-error'?: boolean | string; - 'timeout-minutes'?: number; + 'working-directory'?: string; } interface Job { - name?: string; - needs?: string[]; + env?: Record; + services?: Record; + 'timeout-minutes'?: string | number; if?: string; - 'timeout-minutes'?: number | string; - strategy?: { matrix: { report?: { check: string; shard: number }[]; workspace: string } }; + permissions?: Record; + needs?: string[]; + concurrency?: { group: string; 'cancel-in-progress': boolean; queue?: string }; + outputs?: Record; + strategy?: { matrix: { include?: string; workspace?: string } }; defaults?: { run: { 'working-directory': string } }; steps: Step[]; } interface Workflow { - on: { workflow_dispatch: { inputs: { base: { default: string } } } }; + 'run-name'?: string; + on: { + pull_request?: { paths: string[] }; + issue_comment?: { types: string[] }; + merge_group?: { types: string[] }; + workflow_dispatch: { + inputs: { + base: { default: string }; + scope: { default: string }; + static: { default: string }; + }; + }; + }; permissions: Record; env: Record; jobs: Record; @@ -64,7 +82,7 @@ test('workflow separates tooling, source, installs and artifacts without changin QUALITY_BASE: "${{ github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}", QUALITY_HEAD: '${{ github.event.pull_request.head.sha || github.sha }}', }); - for (const name of ['verify', 'select']) { + for (const name of ['verify', 'select', 'mutation-summary']) { assert.equal(checkouts(workflow.jobs[name]).length, 1); assert.equal(checkouts(workflow.jobs[name])[0].with?.ref, '${{ github.sha }}'); } @@ -93,65 +111,181 @@ test('workflow separates tooling, source, installs and artifacts without changin ], ); assert.equal(runner['working-directory'], undefined); - assert.match(runner.run!, /args=\(\)/); - assert.match(runner.run!, /--shard "\$QUALITY_SHARD"/); - assert.match(runner.run!, /--base "\$QUALITY_BASE" --head "\$QUALITY_HEAD"/); - assert.match( - runner.run!, - /--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/, - ); - const upload = report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6'); + assert.match(runner.run!, /node \.\.\/tooling\/scripts\/quality\/run\.mts/); + assert.equal(report['timeout-minutes'], "${{ matrix.task.check == 'mutation' && 240 || 45 }}"); + assert.equal(workflow.on.workflow_dispatch.inputs.scope.default, 'all'); + assert.equal(workflow.on.workflow_dispatch.inputs.static.default, 'off'); + assert.match(report.env!.QUALITY_STATIC, /inputs\.static \|\| 'off'/); + assert.match(runner.run!, /QUALITY_STATIC:-off/); assert.equal( - upload?.with?.path, - 'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/', + workflow.jobs.select.steps.find(({ id }) => id === 'select')?.env?.QUALITY_SCOPE, + "${{ needs.request.outputs.scope || inputs.scope || 'all' }}", + ); + assert.deepEqual(report.strategy?.matrix, { + include: '${{ fromJSON(needs.select.outputs.reports) }}', + }); + assert.equal(workflow.jobs.select.outputs?.reports, '${{ steps.select.outputs.reports }}'); + assert.equal( + workflow.jobs.select.outputs?.mutationWorkspaces, + '${{ steps.select.outputs.mutationWorkspaces }}', + ); + assert.equal( + workflow.jobs.select.outputs?.hasMutations, + '${{ steps.select.outputs.hasMutations }}', + ); + assert.equal( + workflow.jobs['mutation-summary'].if, + "always() && needs.select.outputs.hasMutations == 'true'", + ); + assert.deepEqual(workflow.jobs['mutation-summary'].strategy?.matrix, { + workspace: '${{ fromJSON(needs.select.outputs.mutationWorkspaces) }}', + }); + assert.equal( + report.steps.find(({ name }) => name === 'Build shared packages')?.env?.QUALITY_WORKSPACE, + '${{ matrix.workspace }}', + ); + assert.equal( + report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path, + "source/coverage/changes/${{ matrix.workspace }}/${{ matrix.task.check }}${{ matrix.task.check == 'mutation' && format('-{0}', matrix.task.shard) || '' }}/", ); - assert.equal(upload?.['continue-on-error'], "${{ matrix.report.check == 'risk' }}"); }); -test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => { - assert.equal(report['timeout-minutes'], "${{ matrix.report.check == 'mutation' && 240 || 45 }}"); - assert.deepEqual(report.strategy?.matrix.report, [ - { check: 'risk', shard: 0 }, - { check: 'mutation', shard: 1 }, - { check: 'mutation', shard: 2 }, - { check: 'mutation', shard: 3 }, - ]); - assert.equal(runner.env?.QUALITY_CHECK, '${{ matrix.report.check }}'); +test('PR updates retain PostgreSQL contracts without starting source quality reports', () => { + const select = workflow.jobs.select; assert.equal( - runner.env?.QUALITY_SHARD, - "${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}", + select.steps.find(({ id }) => id === 'select')?.if, + "github.event_name != 'pull_request'", ); - const aggregate = workflow.jobs.mutation; - assert.equal(aggregate.name, 'Mutation testing (${{ matrix.workspace }})'); - assert.deepEqual(aggregate.needs, ['select', 'reports']); - assert.match(aggregate.if!, /^always\(\)/); - assert.equal(checkouts(aggregate)[0].with?.ref, '${{ github.sha }}'); + const postgresSelection = select.steps.find(({ id }) => id === 'postgres')!; + assert.equal(postgresSelection.if, "github.event_name != 'issue_comment'"); + assert.match(postgresSelection.run!, /git diff --name-only/); + assert.ok(workflow.on.pull_request?.paths.includes('packages/data-schemas/src/postgres/**')); + assert.ok(workflow.on.pull_request?.paths.includes('packages/api/src/app/persistence*')); + const postgres = workflow.jobs.postgres; assert.equal( - aggregate.steps.some(({ run }) => run?.includes('npm ci')), - false, + postgres.if, + "always() && needs.select.result == 'success' && needs.select.outputs.postgresChanged == 'true'", ); - const download = aggregate.steps.find(({ uses }) => uses === 'actions/download-artifact@v7'); + assert.deepEqual(postgres.needs, ['select']); + assert.equal(checkouts(postgres)[0].with?.ref, '${{ env.QUALITY_HEAD }}'); + assert.equal(postgres.services?.postgres.image, 'postgres:16-alpine'); + assert.equal(report.services?.postgres.image, 'postgres:16-alpine'); + assert.ok(postgres.steps.some(({ run }) => run?.includes('npm run test:postgres'))); + assert.ok(postgres.steps.some(({ run }) => run?.includes('delete-user.recovery.spec.js'))); + assert.equal(report.env?.PG_TEST_URL, 'postgresql://postgres@127.0.0.1:5432/postgres'); + const prewarm = report.steps.findIndex( + ({ uses }) => uses === './tooling/.github/actions/mongodb-binary', + ); + assert.ok(prewarm > 0 && prewarm < report.steps.indexOf(runner)); + assert.equal(report.steps[prewarm].with?.['working-directory'], 'source'); +}); + +test('comment publication is trusted and serialized while PR execution stays read-only', (t) => { + assert.ok(workflow.on.pull_request?.paths.includes('scripts/quality/**')); assert.equal( - download?.with?.pattern, - 'change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}', + workflow.jobs.select.if, + "always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true')", ); - assert.equal(download?.with?.['merge-multiple'], undefined); - const verify = aggregate.steps.findIndex(({ name }) => name === 'Verify all mutation shards'); - const requireSuccess = aggregate.steps.findIndex( - ({ name }) => name === 'Require successful report jobs', - ); - assert.ok(verify >= 0 && requireSuccess > verify); - assert.match(aggregate.steps[verify].run!, /--head "\$QUALITY_HEAD"/); - assert.match(aggregate.steps[verify].run!, /--count 3/); - assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success'); - const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6'); - assert.equal(upload?.if, 'always()'); - assert.equal(upload?.['continue-on-error'], true); - assert.equal(upload?.['timeout-minutes'], 3); + assert.deepEqual(workflow.on.issue_comment?.types, ['created']); + assert.equal(workflow.jobs.request.permissions?.issues, 'write'); + assert.match(workflow['run-name']!, /Quality PR #\{0\} request \{1\}/); + for (const name of ['verify', 'select', 'reports', 'mutation-summary']) { + assert.deepEqual(workflow.jobs[name].permissions ?? workflow.permissions, { contents: 'read' }); + } + for (const name of ['reports', 'mutation-summary']) { + const upload = workflow.jobs[name].steps.find( + ({ uses }) => uses === 'actions/upload-artifact@v6', + ); + assert.equal(upload?.with?.overwrite, true, 'reruns replace failed artifacts'); + } + for (const name of ['request', 'progress']) { + const job = workflow.jobs[name]; + assert.equal(checkouts(job)[0].with?.ref, '${{ github.sha }}'); + assert.ok(job.steps.every(({ run }) => !run?.includes('npm'))); + assert.equal(job.permissions?.['pull-requests'], 'write'); + assert.equal(job.permissions?.statuses, 'write'); + assert.equal(job.permissions?.actions, 'write'); + assert.equal(job.concurrency?.group, 'quality-decision-${{ github.event.issue.number }}'); + assert.equal(job.concurrency?.['cancel-in-progress'], false); + assert.equal(job.concurrency?.queue, 'max'); + } + const publisher = yaml.load( + readFileSync(new URL('../../.github/workflows/quality-decision.yml', import.meta.url), 'utf8'), + ) as Workflow; + assert.equal(checkouts(publisher.jobs.publish)[0].with?.ref, '${{ github.workflow_sha }}'); assert.equal( - upload?.with?.name, - 'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}', + publisher.jobs.publish.concurrency?.group, + 'quality-decision-${{ needs.resolve.outputs.pr }}', ); + assert.equal(publisher.jobs.publish.concurrency?.['cancel-in-progress'], false); + assert.equal(publisher.jobs.publish.concurrency?.queue, 'max'); + assert.equal(publisher.permissions.actions, 'write'); + assert.deepEqual(workflow.on.merge_group, { types: ['checks_requested'] }); + assert.equal(publisher.on.merge_group, undefined); + const bridge = workflow.jobs['queue-request']; + assert.equal(bridge.if, "github.event_name == 'merge_group'"); + assert.deepEqual(bridge.permissions ?? workflow.permissions, { contents: 'read' }); + assert.equal(checkouts(bridge).length, 0); + assert.ok(bridge.steps.every(({ run, uses }) => !uses && run?.startsWith('echo '))); + for (const name of ['verify', 'select']) + assert.match(workflow.jobs[name].if!, /github.event_name != 'merge_group'/); + const queue = publisher.jobs.queue; + assert.deepEqual(queue.needs, ['resolve']); + assert.equal(queue.if, "needs.resolve.outputs.queue != ''"); + assert.deepEqual(queue.permissions, { + contents: 'read', + 'pull-requests': 'read', + actions: 'read', + statuses: 'write', + }); + assert.equal(checkouts(queue)[0].with?.ref, '${{ github.workflow_sha }}'); + assert.equal(checkouts(queue)[0].with?.['persist-credentials'], false); + assert.equal(queue.concurrency?.group, 'quality-queue-${{ github.event.workflow_run.head_sha }}'); + assert.ok(queue.steps.some(({ run }) => run === 'node scripts/quality/queue.mts')); + assert.ok( + queue.steps.every( + ({ run, uses }) => !run?.includes('npm') && !uses?.startsWith('actions/download-artifact'), + ), + ); + assert.ok( + publisher.jobs.publish.steps.every( + ({ run, uses }) => !run?.includes('npm') && !uses?.startsWith('actions/download-artifact'), + ), + ); + const directory = mkdtempSync(join(tmpdir(), 'quality-event-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + for (const [event, expected] of [ + [{ pull_request: { number: 12 } }, 'pr=12\n'], + [{ workflow_run: { event: 'merge_group', status: 'completed', id: 55 } }, 'queue=55\n'], + [{ workflow_run: { event: 'merge_group', status: 'in_progress', id: 55 } }, ''], + [ + { workflow_run: { event: 'issue_comment', display_title: 'Quality PR #13 request 99' } }, + 'pr=13\n', + ], + [{ workflow_run: { event: 'pull_request', display_title: 'Quality PR #13 request 99' } }, ''], + [ + { + workflow_run: { + event: 'issue_comment', + display_title: 'Quality PR #13 request $(touch injected)', + }, + }, + '', + ], + ] as const) { + const path = join(directory, 'event.json'); + const output = join(directory, 'output'); + writeFileSync(path, JSON.stringify(event)); + writeFileSync(output, ''); + const result = spawnSync('bash', ['-e', '-c', publisher.jobs.resolve.steps[0].run!], { + cwd: directory, + env: { ...process.env, GITHUB_EVENT_PATH: path, GITHUB_OUTPUT: output }, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(output, 'utf8'), expected); + assert.equal(existsSync(join(directory, 'injected')), false); + } }); for (const olderHead of [true, false]) { @@ -173,26 +307,44 @@ for (const olderHead of [true, false]) { git(['config', 'user.name', 'Quality Test'], upstream); git(['config', 'user.email', 'quality@example.invalid'], upstream); write(upstream, 'package.json', '{"name":"quality-fixture","private":true}'); + write(upstream, 'package-lock.json', '{}'); write(upstream, '.gitignore', 'node_modules/\ncoverage/\n.stryker-tmp/\n'); - const file = 'packages/data-schemas/src/feature.js'; - write(upstream, file, 'module.exports = (value) => value ? 1 : 0;\n'); - write( - upstream, - 'packages/data-schemas/jest.config.mjs', - 'export default { testEnvironment: "node" };', - ); - write( - upstream, - 'packages/data-schemas/src/feature.test.js', - "const feature = require('./feature'); test('uses head dependency', () => { expect(feature(true)).toBe(require('head-only')); expect(feature(false)).toBe(0); });", - ); + const owners = [ + { name: 'data-schemas', path: 'packages/data-schemas', config: 'jest.config.mjs' }, + { name: 'api', path: 'packages/api', config: 'jest.config.mjs' }, + { name: 'server', path: 'api', config: 'jest.config.js' }, + ]; + const mode = olderHead ? 'on' : 'off'; + const sourceCode = + 'module.exports = (value) => value ? 1 : 0;\nmodule.exports.untouched = () => 3;\nmodule.exports.staticValue = (() => 1)();\nfunction hybrid(value) { if (value > 0) return 1; return 0; } module.exports.hybrid = hybrid; module.exports.loaded = hybrid(1);\n'; + for (const owner of owners) { + write(upstream, `${owner.path}/src/feature.js`, sourceCode); + write( + upstream, + `${owner.path}/${owner.config}`, + `${owner.config.endsWith('.mjs') ? 'export default' : 'module.exports ='} { testEnvironment: "node" };`, + ); + write( + upstream, + `${owner.path}/src/feature.test.js`, + "const feature = require('./feature'); test('uses head dependency', () => { expect(feature(true)).toBe(require('head-only')); expect(feature(false)).toBe(0); expect(feature.untouched()).toBe(3); expect(feature.staticValue).toBe(2); expect(feature.hybrid(1)).toBe(2); expect(feature.hybrid(0)).toBe(0); expect(feature.loaded).toBe(2); });", + ); + } const original = commit(); mkdirSync(join(upstream, 'scripts'), { recursive: true }); cpSync(join(root, 'scripts/quality'), join(upstream, 'scripts/quality'), { recursive: true }); cpSync(join(root, 'scripts/crap.mts'), join(upstream, 'scripts/crap.mts')); const base = commit(); git(['checkout', '-qb', 'pr', olderHead ? original : base], upstream); - write(upstream, file, 'module.exports = (value) => value ? 2 : 0;\n'); + for (const owner of owners) + write( + upstream, + `${owner.path}/src/feature.js`, + sourceCode + .replace('? 1', '? 2') + .replaceAll('() => 1', '() => 2') + .replace('return 1', 'return 2'), + ); if (!olderHead) { const selector = readFileSync(join(upstream, 'scripts/quality/changes.mts'), 'utf8'); write( @@ -237,7 +389,12 @@ for (const olderHead of [true, false]) { `${record} require(${JSON.stringify(require.resolve('jest/bin/jest'))});`, ); write(source, 'node_modules/head-only/index.js', 'module.exports = 2;'); - for (const dependency of ['jest-cli', 'jest-config', 'jest-environment-node']) { + for (const dependency of [ + 'jest-cli', + 'jest-config', + 'jest-environment-node', + 'istanbul-lib-coverage', + ]) { symlinkSync( dirname(createRequire(require.resolve('jest')).resolve(`${dependency}/package.json`)), join(source, 'node_modules', dependency), @@ -249,7 +406,7 @@ for (const olderHead of [true, false]) { ...process.env, QUALITY_HEAD: head, QUALITY_WORKSPACE: 'data-schemas', - QUALITY_SHARD: '', + QUALITY_STATIC: mode, GITHUB_OUTPUT: '', GITHUB_STEP_SUMMARY: '', }; @@ -269,7 +426,7 @@ for (const olderHead of [true, false]) { for (const comparison of ['origin/dev', base]) { const output = run(select.run, tooling, { QUALITY_BASE: comparison, - QUALITY_SCOPE: 'data-schemas', + QUALITY_SCOPE: 'all', }); assert.match(output, new RegExp(`"head": "${head}"`)); assert.match(output, new RegExp(`"base": "${olderHead ? original : base}"`)); @@ -281,19 +438,28 @@ for (const olderHead of [true, false]) { const output = run(select.run, tooling, { QUALITY_HEAD: merge, QUALITY_BASE: comparison, - QUALITY_SCOPE: 'data-schemas', + QUALITY_SCOPE: 'all', }); assert.match(output, new RegExp(`"head": "${merge}"`)); assert.match(output, new RegExp(`"base": "${base}"`)); } - for (const check of ['risk', 'mutation']) { + for (const { + workspace, + task: { check, shard }, + } of reportJobs(owners.map(({ name }) => name))) { + const file = `${owners.find(({ name }) => name === workspace)!.path}/src/feature.js`; rmSync(marker, { force: true }); run(runner.run!, source, { QUALITY_BASE: check === 'risk' ? 'origin/dev' : base, + QUALITY_WORKSPACE: workspace, QUALITY_CHECK: check, + QUALITY_SHARD: String(shard), }); - assert.match(readFileSync(marker, 'utf8'), /head Jest/); - const output = join(source, 'coverage/changes/data-schemas', check); + const output = join( + source, + `coverage/changes/${workspace}`, + shard ? `mutation-${shard}` : check, + ); const selection = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8')); assert.equal(selection.head, head); assert.equal(selection.base, olderHead ? original : base); @@ -302,62 +468,87 @@ for (const olderHead of [true, false]) { [file], ); if (check === 'risk') { + assert.match(readFileSync(marker, 'utf8'), /head Jest/); const risk = JSON.parse(readFileSync(join(output, 'risk.json'), 'utf8')); + assert.equal(risk.functions.length, 3, 'unchanged sibling must not be scored'); assert.equal(risk.functions[0].file, file); assert.equal(risk.functions[0].statementCoverage, 1); assert.equal(risk.head, head); } else { - const progress = JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')); - assert.equal(progress.complete, true); - assert.ok(progress.completed > 0); - const unsharded = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')); - const identify = (report: { files: Record }) => - Object.entries(report.files) - .flatMap(([file, { mutants }]) => - mutants.map((value) => { - const mutant = value as { - location: unknown; - mutatorName: string; - replacement: string; - }; - return JSON.stringify([ - file, - mutant.location, - mutant.mutatorName, - mutant.replacement, - ]); - }), - ) - .sort(); - const shardReports = join(directory, 'mutation-reports'); - const selectedMutants: string[] = []; - for (let index = 1; index <= 3; index++) { - run(runner.run!, source, { - QUALITY_BASE: base, - QUALITY_CHECK: 'mutation', - QUALITY_SHARD: `${index}/3`, - }); - const shard = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8')); - assert.deepEqual(shard.shard, { index, total: 3 }); - assert.equal(shard.workspace, 'data-schemas'); - const mutation = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')); - selectedMutants.push(...identify(mutation)); - cpSync(output, join(shardReports, `shard-${index}`), { recursive: true }); + const completion = JSON.parse(readFileSync(join(output, 'completion.json'), 'utf8')); + assert.equal(completion.complete, true); + assert.equal(completion.index, shard); + if (completion.total) { + assert.match(readFileSync(marker, 'utf8'), /head Jest/); + const progress = JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')); + assert.equal(progress.complete, true); + assert.equal(progress.staticMutants, mode === 'on'); + assert.equal(progress.completed, completion.total); } - assert.deepEqual(selectedMutants.sort(), identify(unsharded)); - const aggregateOutput = join(directory, 'aggregate'); - run( - `node ../tooling/scripts/quality/aggregate.mts --directory "${shardReports}" --head "${head}" --workspace data-schemas --count 3 --output "${aggregateOutput}"`, - source, - {}, + cpSync( + output, + join(tooling, `coverage/shards/change-mutation-data-schemas-${head}-shard-${shard}`), + { recursive: true }, ); - const aggregate = JSON.parse(readFileSync(join(aggregateOutput, 'aggregate.json'), 'utf8')); - assert.equal(aggregate.complete, true); - assert.equal(aggregate.head, head); - assert.equal(aggregate.total, selectedMutants.length); } assert.equal(git(['rev-parse', 'HEAD'], source).trim(), head); assert.equal(git(['diff', 'HEAD', '--', file], source), ''); } + const combine = workflow.jobs['mutation-summary'].steps.find( + ({ name }) => name === 'Verify and combine mutation results', + )!; + run(combine.run!, tooling, {}); + const combined = JSON.parse( + readFileSync(join(tooling, 'coverage/changes/data-schemas/mutation/mutation.json'), 'utf8'), + ); + const mutants: MutationReport['files'][string]['mutants'] = + combined.files['packages/data-schemas/src/feature.js'].mutants; + console.log( + JSON.stringify({ + fixture: 'exact-head sharded Stryker', + static: mode, + planned: mutants.length, + ignored: mutants.filter(({ status }) => status === 'Ignored').length, + tested: mutants.filter(({ status }) => + ['Killed', 'Timeout', 'Survived', 'NoCoverage'].includes(status), + ).length, + }), + ); + assert.ok(mutants.length > 0); + assert.equal( + mutants.some((mutant: { status: string }) => mutant.status === 'Ignored'), + mode === 'off', + ); + assert.ok( + mutants.some( + (mutant) => + mutant.static && mutant.location.start.line === 4 && mutant.status !== 'Ignored', + ), + `hybrid mutants remain tested: ${JSON.stringify(mutants)}`, + ); + run( + 'node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --workspace data-schemas --check mutation --static "$QUALITY_STATIC"', + source, + { QUALITY_BASE: base }, + ); + const unsharded = JSON.parse( + readFileSync(join(source, 'coverage/changes/data-schemas/mutation/mutation.json'), 'utf8'), + ); + const keys = (report: MutationReport) => + Object.entries(report.files).flatMap(([file, value]) => + value.mutants.map((mutant) => + mutantKey(file, { ...mutant, replacement: mutant.replacement! }), + ), + ); + assertMutants(keys(unsharded), keys(combined)); + assert.equal( + mutationSummary(unsharded, 'data-schemas', mode === 'on'), + mutationSummary(combined, 'data-schemas', mode === 'on'), + ); + + assert.match( + readFileSync(join(tooling, 'coverage/changes/data-schemas/mutation/summary.md'), 'utf8'), + /All four shards completed/, + ); }); }