mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 14:08:21 +00:00
🧯 ci: Stop Diagnostic Artifact Uploads From Failing or Stalling Test Jobs (#18068)
On 2026-11-11 the artifact service answered every request with an HTML page. In Playwright run 43092661470 eight memory shards failed in "Upload Playwright HTML report" after retrying for 9 to 10.5 minutes; in six of them every test had passed. Uploads that no job downloads (Playwright reports, traces and screenshots, the Lighthouse report, the advisory quality reports) are now `continue-on-error` with a three minute step timeout, so a failed upload stays out of the job result and a stalled one cannot run the job into its own timeout. The Jest result uploads were already `continue-on-error` and gain the same bound. Build outputs and mutation shards that a later job downloads stay fatal. merge-gate.test.mjs holds the gated workflows to that split, and workflow.test.mts holds Change Quality to it.
This commit is contained in:
parent
d434417089
commit
bd9108a87a
8 changed files with 87 additions and 7 deletions
21
.github/scripts/merge-gate.test.mjs
vendored
21
.github/scripts/merge-gate.test.mjs
vendored
|
|
@ -138,6 +138,27 @@ test('every gated workflow starts for a merge group and closes in one gate over
|
|||
}
|
||||
});
|
||||
|
||||
test('an artifact upload can fail a gated job only when another job downloads the artifact', () => {
|
||||
const runs = (step, action) => step.uses?.startsWith(`actions/${action}-artifact@`);
|
||||
for (const suite of SUITES) {
|
||||
const steps = Object.entries(workflows[suite.workflow].jobs).flatMap(([job, definition]) =>
|
||||
definition.steps.map((step) => ({ job, step })),
|
||||
);
|
||||
const downloaded = new Set(
|
||||
steps.filter(({ step }) => runs(step, 'download')).map(({ step }) => step.with.name),
|
||||
);
|
||||
for (const { job, step } of steps.filter(({ step }) => runs(step, 'upload'))) {
|
||||
const label = `${suite.workflow} ${job} uploads ${step.with.name}`;
|
||||
if (downloaded.has(step.with.name)) {
|
||||
assert.equal(step['continue-on-error'], undefined, `${label}, which a later job needs`);
|
||||
continue;
|
||||
}
|
||||
assert.equal(step['continue-on-error'], true, `${label}, which no job downloads`);
|
||||
assert.ok(step['timeout-minutes'] > 0, `${label} without a bound on a stalled upload`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a required context is the name of exactly one gate job and of nothing else', () => {
|
||||
const contexts = SUITES.map((suite) => suite.context);
|
||||
assert.equal(new Set(contexts).size, contexts.length);
|
||||
|
|
|
|||
16
.github/workflows/backend-review.yml
vendored
16
.github/workflows/backend-review.yml
vendored
|
|
@ -518,9 +518,11 @@ jobs:
|
|||
npm run test:ci -- --shard=${{ matrix.shard }}/3 $JEST_JSON
|
||||
fi
|
||||
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
|
||||
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
|
||||
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
|
||||
# overwrites its own artifact.
|
||||
- name: Upload Jest results
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
|
@ -590,9 +592,11 @@ jobs:
|
|||
npm run test:ci -- $JEST_JSON
|
||||
fi
|
||||
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
|
||||
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
|
||||
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
|
||||
# overwrites its own artifact.
|
||||
- name: Upload Jest results
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
|
@ -676,9 +680,11 @@ jobs:
|
|||
npm run test:ci -- $JEST_JSON
|
||||
fi
|
||||
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
|
||||
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
|
||||
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
|
||||
# overwrites its own artifact.
|
||||
- name: Upload Jest results
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
|
@ -776,9 +782,11 @@ jobs:
|
|||
npm run test:ci -- --shard=${{ matrix.shard }}/4 $JEST_JSON
|
||||
fi
|
||||
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
|
||||
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
|
||||
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
|
||||
# overwrites its own artifact.
|
||||
- name: Upload Jest results
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
|
|
|||
7
.github/workflows/change-quality.yml
vendored
7
.github/workflows/change-quality.yml
vendored
|
|
@ -204,8 +204,11 @@ jobs:
|
|||
if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi
|
||||
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \
|
||||
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}"
|
||||
# The mutation job downloads the mutation shards, so a shard that is not uploaded has to
|
||||
# fail here. Nothing downloads the risk report, so losing it must not fail the job.
|
||||
- name: Upload reports
|
||||
if: always()
|
||||
continue-on-error: ${{ matrix.report.check == 'risk' }}
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }}
|
||||
|
|
@ -246,8 +249,12 @@ jobs:
|
|||
env:
|
||||
REPORT_RESULT: ${{ needs.reports.result }}
|
||||
run: test "$REPORT_RESULT" = success
|
||||
# The aggregate is a debugging aid that no job downloads: see the note on the e2e_shards
|
||||
# uploads in playwright-mock.yml.
|
||||
- uses: actions/upload-artifact@v6
|
||||
if: always()
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}
|
||||
path: coverage/changes/${{ matrix.workspace }}/mutation/
|
||||
|
|
|
|||
4
.github/workflows/favorites-quality.yml
vendored
4
.github/workflows/favorites-quality.yml
vendored
|
|
@ -80,8 +80,12 @@ jobs:
|
|||
run: npm run test:quality:${{ matrix.suite }} --workspace packages/data-schemas
|
||||
- name: Mutate the real database implementation
|
||||
run: npm run test:mutations:${{ matrix.suite }} --workspace packages/data-schemas
|
||||
# A debugging aid that no job downloads, so a failed or stalled upload must not fail the
|
||||
# contracts: see the note on the e2e_shards uploads in playwright-mock.yml.
|
||||
- name: Upload advisory reports
|
||||
if: always()
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ${{ matrix.suite }}-quality-${{ github.sha }}
|
||||
|
|
|
|||
8
.github/workflows/frontend-review.yml
vendored
8
.github/workflows/frontend-review.yml
vendored
|
|
@ -502,9 +502,11 @@ jobs:
|
|||
fi
|
||||
working-directory: packages/client
|
||||
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
|
||||
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
|
||||
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
|
||||
# overwrites its own artifact.
|
||||
- name: Upload Jest results
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
|
@ -592,9 +594,11 @@ jobs:
|
|||
fi
|
||||
working-directory: client
|
||||
# Per-test results keyed by head SHA (run.head_sha), for the codegraph test-evidence feed.
|
||||
# Never part of the gate: it cannot fail the job, and a re-run overwrites its own artifact.
|
||||
# Never part of the gate: it cannot fail the job or stall it into its timeout, and a re-run
|
||||
# overwrites its own artifact.
|
||||
- name: Upload Jest results
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
|
|
|||
4
.github/workflows/lighthouse.yml
vendored
4
.github/workflows/lighthouse.yml
vendored
|
|
@ -118,8 +118,12 @@ jobs:
|
|||
echo "[Open the full run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)"
|
||||
} > lighthouse-comment.md
|
||||
gh pr comment "$PR_NUMBER" --body-file lighthouse-comment.md
|
||||
# A debugging aid that no job downloads, so a failed or stalled upload must not fail the
|
||||
# audit: see the note on the e2e_shards uploads in playwright-mock.yml.
|
||||
- name: Upload Lighthouse reports
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: lighthouse
|
||||
|
|
|
|||
26
.github/workflows/playwright-mock.yml
vendored
26
.github/workflows/playwright-mock.yml
vendored
|
|
@ -424,8 +424,18 @@ jobs:
|
|||
CI: 'true'
|
||||
E2E_ALLOW_EMAIL_CHANGE: 'false'
|
||||
|
||||
# Reports and traces are debugging aids that no job downloads, so an upload that fails must
|
||||
# not fail a shard whose tests passed. On 2026-11-11 the artifact service answered every
|
||||
# request with an HTML page: each upload retried for 9 to 10.5 minutes before it failed,
|
||||
# six shards whose tests had passed went red, and one of them ended within seconds of the
|
||||
# job's 30 minute timeout. `continue-on-error` keeps a failed upload out of the job result,
|
||||
# and the bound keeps a stalled one from running the job into that timeout: the slowest of
|
||||
# 519 successful report uploads took 10 seconds. The runner still annotates the run with
|
||||
# the upload's own error, or with the step timeout.
|
||||
- name: Upload Playwright HTML report
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-report-${{ matrix.artifact }}
|
||||
|
|
@ -435,6 +445,8 @@ jobs:
|
|||
|
||||
- name: Upload traces & screenshots
|
||||
if: failure()
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-test-results-${{ matrix.artifact }}
|
||||
|
|
@ -455,6 +467,8 @@ jobs:
|
|||
|
||||
- name: Upload title ownership diagnostics
|
||||
if: ${{ always() && matrix.stream_store == 'memory' && startsWith(matrix.shard, '1/') }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-title-ownership-${{ matrix.artifact }}
|
||||
|
|
@ -599,8 +613,12 @@ jobs:
|
|||
npx playwright test --config=e2e/playwright.config.mock.ts
|
||||
mcp-tool-list-changed.spec.ts --retries=0
|
||||
|
||||
# Debugging aids that no job downloads: non-fatal and bounded, as the note on the
|
||||
# e2e_shards uploads explains.
|
||||
- name: Upload Playwright HTML report
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-mcp-list-changed-${{ matrix.replicas }}-replicas
|
||||
|
|
@ -610,6 +628,8 @@ jobs:
|
|||
|
||||
- name: Upload traces & screenshots
|
||||
if: failure()
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-mcp-list-changed-results-${{ matrix.replicas }}-replicas
|
||||
|
|
@ -686,8 +706,12 @@ jobs:
|
|||
- name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)
|
||||
run: node e2e/mcp-apps/run.mjs
|
||||
|
||||
# Debugging aids that no job downloads: non-fatal and bounded, as the note on the
|
||||
# e2e_shards uploads explains.
|
||||
- name: Upload per-phase Playwright reports
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-mcp-apps-reports
|
||||
|
|
@ -699,6 +723,8 @@ jobs:
|
|||
|
||||
- name: Upload MCP Apps traces and screenshots
|
||||
if: failure()
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-mcp-apps-results
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ interface Step {
|
|||
'working-directory'?: string;
|
||||
if?: string;
|
||||
env?: Record<string, string>;
|
||||
'continue-on-error'?: boolean | string;
|
||||
'timeout-minutes'?: number;
|
||||
}
|
||||
interface Job {
|
||||
name?: string;
|
||||
|
|
@ -98,10 +100,12 @@ test('workflow separates tooling, source, installs and artifacts without changin
|
|||
runner.run!,
|
||||
/--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/,
|
||||
);
|
||||
const upload = report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6');
|
||||
assert.equal(
|
||||
report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path,
|
||||
upload?.with?.path,
|
||||
'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/',
|
||||
);
|
||||
assert.equal(upload?.['continue-on-error'], "${{ matrix.report.check == 'risk' }}");
|
||||
});
|
||||
|
||||
test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => {
|
||||
|
|
@ -142,6 +146,8 @@ test('mutation shards retain the fixed budget and aggregate all evidence before
|
|||
assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success');
|
||||
const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6');
|
||||
assert.equal(upload?.if, 'always()');
|
||||
assert.equal(upload?.['continue-on-error'], true);
|
||||
assert.equal(upload?.['timeout-minutes'], 3);
|
||||
assert.equal(
|
||||
upload?.with?.name,
|
||||
'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue