From bd9108a87a020e5e59d708dd208962363fcd96aa Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 11 Nov 2026 19:17:15 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AF=20ci:=20Stop=20Diagnostic=20Artifa?= =?UTF-8?q?ct=20Uploads=20From=20Failing=20or=20Stalling=20Test=20Jobs=20(?= =?UTF-8?q?#18068)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 2026-11-11 the artifact service answered every request with an HTML page. In Playwright run 43092661470 eight memory shards failed in "Upload Playwright HTML report" after retrying for 9 to 10.5 minutes; in six of them every test had passed. Uploads that no job downloads (Playwright reports, traces and screenshots, the Lighthouse report, the advisory quality reports) are now `continue-on-error` with a three minute step timeout, so a failed upload stays out of the job result and a stalled one cannot run the job into its own timeout. The Jest result uploads were already `continue-on-error` and gain the same bound. Build outputs and mutation shards that a later job downloads stay fatal. merge-gate.test.mjs holds the gated workflows to that split, and workflow.test.mts holds Change Quality to it. --- .github/scripts/merge-gate.test.mjs | 21 ++++++++++++++++++++ .github/workflows/backend-review.yml | 16 +++++++++++---- .github/workflows/change-quality.yml | 7 +++++++ .github/workflows/favorites-quality.yml | 4 ++++ .github/workflows/frontend-review.yml | 8 ++++++-- .github/workflows/lighthouse.yml | 4 ++++ .github/workflows/playwright-mock.yml | 26 +++++++++++++++++++++++++ scripts/quality/workflow.test.mts | 8 +++++++- 8 files changed, 87 insertions(+), 7 deletions(-) diff --git a/.github/scripts/merge-gate.test.mjs b/.github/scripts/merge-gate.test.mjs index 3a0e6acb60..5ce2f1096a 100644 --- a/.github/scripts/merge-gate.test.mjs +++ b/.github/scripts/merge-gate.test.mjs @@ -138,6 +138,27 @@ test('every gated workflow starts for a merge group and closes in one gate over } }); +test('an artifact upload can fail a gated job only when another job downloads the artifact', () => { + const runs = (step, action) => step.uses?.startsWith(`actions/${action}-artifact@`); + for (const suite of SUITES) { + const steps = Object.entries(workflows[suite.workflow].jobs).flatMap(([job, definition]) => + definition.steps.map((step) => ({ job, step })), + ); + const downloaded = new Set( + steps.filter(({ step }) => runs(step, 'download')).map(({ step }) => step.with.name), + ); + for (const { job, step } of steps.filter(({ step }) => runs(step, 'upload'))) { + const label = `${suite.workflow} ${job} uploads ${step.with.name}`; + if (downloaded.has(step.with.name)) { + assert.equal(step['continue-on-error'], undefined, `${label}, which a later job needs`); + continue; + } + assert.equal(step['continue-on-error'], true, `${label}, which no job downloads`); + assert.ok(step['timeout-minutes'] > 0, `${label} without a bound on a stalled upload`); + } + } +}); + test('a required context is the name of exactly one gate job and of nothing else', () => { const contexts = SUITES.map((suite) => suite.context); assert.equal(new Set(contexts).size, contexts.length); diff --git a/.github/workflows/backend-review.yml b/.github/workflows/backend-review.yml index 1888f71f60..bca107f6dc 100644 --- a/.github/workflows/backend-review.yml +++ b/.github/workflows/backend-review.yml @@ -518,9 +518,11 @@ jobs: npm run test:ci -- --shard=${{ matrix.shard }}/3 $JEST_JSON fi # Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed. - # Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact. + # Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run + # overwrites its own artifact. - name: Upload Jest results if: ${{ !cancelled() }} + timeout-minutes: 3 continue-on-error: true uses: actions/upload-artifact@v6 with: @@ -590,9 +592,11 @@ jobs: npm run test:ci -- $JEST_JSON fi # Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed. - # Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact. + # Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run + # overwrites its own artifact. - name: Upload Jest results if: ${{ !cancelled() }} + timeout-minutes: 3 continue-on-error: true uses: actions/upload-artifact@v6 with: @@ -676,9 +680,11 @@ jobs: npm run test:ci -- $JEST_JSON fi # Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed. - # Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact. + # Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run + # overwrites its own artifact. - name: Upload Jest results if: ${{ !cancelled() }} + timeout-minutes: 3 continue-on-error: true uses: actions/upload-artifact@v6 with: @@ -776,9 +782,11 @@ jobs: npm run test:ci -- --shard=${{ matrix.shard }}/4 $JEST_JSON fi # Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed. - # Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact. + # Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run + # overwrites its own artifact. - name: Upload Jest results if: ${{ !cancelled() }} + timeout-minutes: 3 continue-on-error: true uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/change-quality.yml b/.github/workflows/change-quality.yml index 92a4829146..e6dc959db9 100644 --- a/.github/workflows/change-quality.yml +++ b/.github/workflows/change-quality.yml @@ -204,8 +204,11 @@ jobs: if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \ --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}" + # The mutation job downloads the mutation shards, so a shard that is not uploaded has to + # fail here. Nothing downloads the risk report, so losing it must not fail the job. - name: Upload reports if: always() + continue-on-error: ${{ matrix.report.check == 'risk' }} uses: actions/upload-artifact@v6 with: name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }} @@ -246,8 +249,12 @@ jobs: env: REPORT_RESULT: ${{ needs.reports.result }} run: test "$REPORT_RESULT" = success + # The aggregate is a debugging aid that no job downloads: see the note on the e2e_shards + # uploads in playwright-mock.yml. - uses: actions/upload-artifact@v6 if: always() + timeout-minutes: 3 + continue-on-error: true with: name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }} path: coverage/changes/${{ matrix.workspace }}/mutation/ diff --git a/.github/workflows/favorites-quality.yml b/.github/workflows/favorites-quality.yml index 5de5da2a57..c424018f06 100644 --- a/.github/workflows/favorites-quality.yml +++ b/.github/workflows/favorites-quality.yml @@ -80,8 +80,12 @@ jobs: run: npm run test:quality:${{ matrix.suite }} --workspace packages/data-schemas - name: Mutate the real database implementation run: npm run test:mutations:${{ matrix.suite }} --workspace packages/data-schemas + # A debugging aid that no job downloads, so a failed or stalled upload must not fail the + # contracts: see the note on the e2e_shards uploads in playwright-mock.yml. - name: Upload advisory reports if: always() + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: ${{ matrix.suite }}-quality-${{ github.sha }} diff --git a/.github/workflows/frontend-review.yml b/.github/workflows/frontend-review.yml index c347cedeab..83ccd455de 100644 --- a/.github/workflows/frontend-review.yml +++ b/.github/workflows/frontend-review.yml @@ -502,9 +502,11 @@ jobs: fi working-directory: packages/client # Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed. - # Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact. + # Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run + # overwrites its own artifact. - name: Upload Jest results if: ${{ !cancelled() }} + timeout-minutes: 3 continue-on-error: true uses: actions/upload-artifact@v6 with: @@ -592,9 +594,11 @@ jobs: fi working-directory: client # Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed. - # Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact. + # Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run + # overwrites its own artifact. - name: Upload Jest results if: ${{ !cancelled() }} + timeout-minutes: 3 continue-on-error: true uses: actions/upload-artifact@v6 with: diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index 77f4dba601..610d093c07 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -118,8 +118,12 @@ jobs: echo "[Open the full run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" } > lighthouse-comment.md gh pr comment "$PR_NUMBER" --body-file lighthouse-comment.md + # A debugging aid that no job downloads, so a failed or stalled upload must not fail the + # audit: see the note on the e2e_shards uploads in playwright-mock.yml. - name: Upload Lighthouse reports if: ${{ !cancelled() }} + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: lighthouse diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 3d4c65e900..b2238689e3 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -424,8 +424,18 @@ jobs: CI: 'true' E2E_ALLOW_EMAIL_CHANGE: 'false' + # Reports and traces are debugging aids that no job downloads, so an upload that fails must + # not fail a shard whose tests passed. On 2026-11-11 the artifact service answered every + # request with an HTML page: each upload retried for 9 to 10.5 minutes before it failed, + # six shards whose tests had passed went red, and one of them ended within seconds of the + # job's 30 minute timeout. `continue-on-error` keeps a failed upload out of the job result, + # and the bound keeps a stalled one from running the job into that timeout: the slowest of + # 519 successful report uploads took 10 seconds. The runner still annotates the run with + # the upload's own error, or with the step timeout. - name: Upload Playwright HTML report if: ${{ !cancelled() }} + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: playwright-report-${{ matrix.artifact }} @@ -435,6 +445,8 @@ jobs: - name: Upload traces & screenshots if: failure() + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: playwright-test-results-${{ matrix.artifact }} @@ -455,6 +467,8 @@ jobs: - name: Upload title ownership diagnostics if: ${{ always() && matrix.stream_store == 'memory' && startsWith(matrix.shard, '1/') }} + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: playwright-title-ownership-${{ matrix.artifact }} @@ -599,8 +613,12 @@ jobs: npx playwright test --config=e2e/playwright.config.mock.ts mcp-tool-list-changed.spec.ts --retries=0 + # Debugging aids that no job downloads: non-fatal and bounded, as the note on the + # e2e_shards uploads explains. - name: Upload Playwright HTML report if: ${{ !cancelled() }} + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v4 with: name: playwright-mcp-list-changed-${{ matrix.replicas }}-replicas @@ -610,6 +628,8 @@ jobs: - name: Upload traces & screenshots if: failure() + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v4 with: name: playwright-mcp-list-changed-results-${{ matrix.replicas }}-replicas @@ -686,8 +706,12 @@ jobs: - name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota) run: node e2e/mcp-apps/run.mjs + # Debugging aids that no job downloads: non-fatal and bounded, as the note on the + # e2e_shards uploads explains. - name: Upload per-phase Playwright reports if: ${{ !cancelled() }} + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: playwright-mcp-apps-reports @@ -699,6 +723,8 @@ jobs: - name: Upload MCP Apps traces and screenshots if: failure() + timeout-minutes: 3 + continue-on-error: true uses: actions/upload-artifact@v6 with: name: playwright-mcp-apps-results diff --git a/scripts/quality/workflow.test.mts b/scripts/quality/workflow.test.mts index 95af5e458d..b20d6173bf 100644 --- a/scripts/quality/workflow.test.mts +++ b/scripts/quality/workflow.test.mts @@ -28,6 +28,8 @@ interface Step { 'working-directory'?: string; if?: string; env?: Record; + 'continue-on-error'?: boolean | string; + 'timeout-minutes'?: number; } interface Job { name?: string; @@ -98,10 +100,12 @@ test('workflow separates tooling, source, installs and artifacts without changin runner.run!, /--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/, ); + const upload = report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6'); assert.equal( - report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path, + upload?.with?.path, 'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/', ); + assert.equal(upload?.['continue-on-error'], "${{ matrix.report.check == 'risk' }}"); }); test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => { @@ -142,6 +146,8 @@ test('mutation shards retain the fixed budget and aggregate all evidence before assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success'); const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6'); assert.equal(upload?.if, 'always()'); + assert.equal(upload?.['continue-on-error'], true); + assert.equal(upload?.['timeout-minutes'], 3); assert.equal( upload?.with?.name, 'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}',