🧯 ci: Stop Diagnostic Artifact Uploads From Failing or Stalling Test Jobs (#18068)

On 2026-11-11 the artifact service answered every request with an HTML
page. In Playwright run 43092661470 eight memory shards failed in
"Upload Playwright HTML report" after retrying for 9 to 10.5 minutes;
in six of them every test had passed.

Uploads that no job downloads (Playwright reports, traces and
screenshots, the Lighthouse report, the advisory quality reports) are
now `continue-on-error` with a three minute step timeout, so a failed
upload stays out of the job result and a stalled one cannot run the job
into its own timeout. The Jest result uploads were already
`continue-on-error` and gain the same bound. Build outputs and mutation
shards that a later job downloads stay fatal.

merge-gate.test.mjs holds the gated workflows to that split, and
workflow.test.mts holds Change Quality to it.
This commit is contained in:
Danny Avila 2026-11-11 19:17:15 -05:00 • committed by GitHub
parent d434417089
commit bd9108a87a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 87 additions and 7 deletions

View file

@ -138,6 +138,27 @@ test('every gated workflow starts for a merge group and closes in one gate over
}
});
test('an artifact upload can fail a gated job only when another job downloads the artifact', () => {
const runs = (step, action) => step.uses?.startsWith(`actions/${action}-artifact@`);
for (const suite of SUITES) {
const steps = Object.entries(workflows[suite.workflow].jobs).flatMap(([job, definition]) =>
definition.steps.map((step) => ({ job, step })),
);
const downloaded = new Set(
steps.filter(({ step }) => runs(step, 'download')).map(({ step }) => step.with.name),
);
for (const { job, step } of steps.filter(({ step }) => runs(step, 'upload'))) {
const label = `${suite.workflow} ${job} uploads ${step.with.name}`;
if (downloaded.has(step.with.name)) {
assert.equal(step['continue-on-error'], undefined, `${label}, which a later job needs`);
continue;
}
assert.equal(step['continue-on-error'], true, `${label}, which no job downloads`);
assert.ok(step['timeout-minutes'] > 0, `${label} without a bound on a stalled upload`);
}
}
});
test('a required context is the name of exactly one gate job and of nothing else', () => {
const contexts = SUITES.map((suite) => suite.context);
assert.equal(new Set(contexts).size, contexts.length);

View file

@ -518,9 +518,11 @@ jobs:
npm run test:ci -- --shard=${{ matrix.shard }}/3 $JEST_JSON
fi
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
# overwrites its own artifact.
- name: Upload Jest results
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
@ -590,9 +592,11 @@ jobs:
npm run test:ci -- $JEST_JSON
fi
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
# overwrites its own artifact.
- name: Upload Jest results
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
@ -676,9 +680,11 @@ jobs:
npm run test:ci -- $JEST_JSON
fi
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
# overwrites its own artifact.
- name: Upload Jest results
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
@ -776,9 +782,11 @@ jobs:
npm run test:ci -- --shard=${{ matrix.shard }}/4 $JEST_JSON
fi
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
# overwrites its own artifact.
- name: Upload Jest results
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:

View file

@ -204,8 +204,11 @@ jobs:
if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}"
# The mutation job downloads the mutation shards, so a shard that is not uploaded has to
# fail here. Nothing downloads the risk report, so losing it must not fail the job.
- name: Upload reports
if: always()
continue-on-error: ${{ matrix.report.check == 'risk' }}
uses: actions/upload-artifact@v6
with:
name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }}
@ -246,8 +249,12 @@ jobs:
env:
REPORT_RESULT: ${{ needs.reports.result }}
run: test "$REPORT_RESULT" = success
# The aggregate is a debugging aid that no job downloads: see the note on the e2e_shards
# uploads in playwright-mock.yml.
- uses: actions/upload-artifact@v6
if: always()
timeout-minutes: 3
continue-on-error: true
with:
name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}
path: coverage/changes/${{ matrix.workspace }}/mutation/

View file

@ -80,8 +80,12 @@ jobs:
run: npm run test:quality:${{ matrix.suite }} --workspace packages/data-schemas
- name: Mutate the real database implementation
run: npm run test:mutations:${{ matrix.suite }} --workspace packages/data-schemas
# A debugging aid that no job downloads, so a failed or stalled upload must not fail the
# contracts: see the note on the e2e_shards uploads in playwright-mock.yml.
- name: Upload advisory reports
if: always()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: ${{ matrix.suite }}-quality-${{ github.sha }}

View file

@ -502,9 +502,11 @@ jobs:
fi
working-directory: packages/client
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
# overwrites its own artifact.
- name: Upload Jest results
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
@ -592,9 +594,11 @@ jobs:
fi
working-directory: client
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
# overwrites its own artifact.
- name: Upload Jest results
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:

View file

@ -118,8 +118,12 @@ jobs:
echo "[Open the full run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)"
} > lighthouse-comment.md
gh pr comment "$PR_NUMBER" --body-file lighthouse-comment.md
# A debugging aid that no job downloads, so a failed or stalled upload must not fail the
# audit: see the note on the e2e_shards uploads in playwright-mock.yml.
- name: Upload Lighthouse reports
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: lighthouse

View file

@ -424,8 +424,18 @@ jobs:
CI: 'true'
E2E_ALLOW_EMAIL_CHANGE: 'false'
# Reports and traces are debugging aids that no job downloads, so an upload that fails must
# not fail a shard whose tests passed. On 2026-11-11 the artifact service answered every
# request with an HTML page: each upload retried for 9 to 10.5 minutes before it failed,
# six shards whose tests had passed went red, and one of them ended within seconds of the
# job's 30 minute timeout. `continue-on-error` keeps a failed upload out of the job result,
# and the bound keeps a stalled one from running the job into that timeout: the slowest of
# 519 successful report uploads took 10 seconds. The runner still annotates the run with
# the upload's own error, or with the step timeout.
- name: Upload Playwright HTML report
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-report-${{ matrix.artifact }}
@ -435,6 +445,8 @@ jobs:
- name: Upload traces & screenshots
if: failure()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-test-results-${{ matrix.artifact }}
@ -455,6 +467,8 @@ jobs:
- name: Upload title ownership diagnostics
if: ${{ always() && matrix.stream_store == 'memory' && startsWith(matrix.shard, '1/') }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-title-ownership-${{ matrix.artifact }}
@ -599,8 +613,12 @@ jobs:
npx playwright test --config=e2e/playwright.config.mock.ts
mcp-tool-list-changed.spec.ts --retries=0
# Debugging aids that no job downloads: non-fatal and bounded, as the note on the
# e2e_shards uploads explains.
- name: Upload Playwright HTML report
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: playwright-mcp-list-changed-${{ matrix.replicas }}-replicas
@ -610,6 +628,8 @@ jobs:
- name: Upload traces & screenshots
if: failure()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: playwright-mcp-list-changed-results-${{ matrix.replicas }}-replicas
@ -686,8 +706,12 @@ jobs:
- name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)
run: node e2e/mcp-apps/run.mjs
# Debugging aids that no job downloads: non-fatal and bounded, as the note on the
# e2e_shards uploads explains.
- name: Upload per-phase Playwright reports
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-mcp-apps-reports
@ -699,6 +723,8 @@ jobs:
- name: Upload MCP Apps traces and screenshots
if: failure()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-mcp-apps-results

View file

@ -28,6 +28,8 @@ interface Step {
'working-directory'?: string;
if?: string;
env?: Record<string, string>;
'continue-on-error'?: boolean | string;
'timeout-minutes'?: number;
}
interface Job {
name?: string;
@ -98,10 +100,12 @@ test('workflow separates tooling, source, installs and artifacts without changin
runner.run!,
/--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/,
);
const upload = report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6');
assert.equal(
report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path,
upload?.with?.path,
'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/',
);
assert.equal(upload?.['continue-on-error'], "${{ matrix.report.check == 'risk' }}");
});
test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => {
@ -142,6 +146,8 @@ test('mutation shards retain the fixed budget and aggregate all evidence before
assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success');
const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6');
assert.equal(upload?.if, 'always()');
assert.equal(upload?.['continue-on-error'], true);
assert.equal(upload?.['timeout-minutes'], 3);
assert.equal(
upload?.with?.name,
'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}',