From d32403f851cbb34d4637f3a27df480e25418a7d0 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 11 Nov 2026 10:31:46 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=8D=83=20ci:=20Provision=20the=20mongod?= =?UTF-8?q?=20Binary=20Before=20Any=20Test=20Starts=20(#18053)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ๐Ÿƒ ci: Provision the mongod Binary Before Tests Start mongodb-memory-server downloads its ~122 MB mongod the first time a test starts a server, inside a 15 s beforeAll. Only backend-review.yml cached ~/.cache/mongodb-binaries (the same step pasted four times), and a cache alone still leaves the download inside the first hook on a miss. Integration Tests downloaded on every run: 6 failures in 274 runs with "Exceeded timeout of 15000 ms for a hook" in secrets.integration.spec.ts, followed by "spawn ETXTBSY" in the suites that started mongod while the timed-out download was still writing it. Add .github/actions/mongodb-binary: restore the cache with the existing key scheme, ask the installed mongodb-memory-server for the binary it wants (MongoBinary.getPath), make sure that binary sits in the home cache every working directory resolves, run it once, and save the cache right away. The step fails the job when no runnable binary can be put in place. Use it in every job that starts mongodb-memory-server, after dependencies are installed and before tests, and add the action to the path filters of the workflows that list their own file. * ๐Ÿงช test: Cover the mongod Provisioning Script Without the Network Run the real script against a stand-in resolver that keeps the properties it relies on: home cache first, download when missing, and a memoised path until the cache is cleared. Integration Tests runs the suite before it uses the action. * ๐Ÿงช test: Run mongod Provisioning Against the Real Resolver The suite replaced mongodb-memory-server with a hand-written stand-in, so it checked the script against a description of the resolver rather than the installed one. It now runs the script against the real library and replaces only the network: MONGOMS_DOWNLOAD_MIRROR points at a local server that serves an archive whose bin/mongod is a shell script, with the .md5 the library verifies it against. Recovery also no longer charges a broken copy that was already on disk to the download: with a broken binary in both the home cache and the root node_modules/.cache, the first attempt removed one, the retry met the other, and the step failed without ever downloading. Existing copies are now discarded one at a time and the download keeps its own two attempts. * ๐Ÿงช test: Reach the Exit-Status Check Behind the Library's System Binary Probe A system binary that prints nothing is rejected by the library's own version probe, so the case never reached this script's check or the rule that a file the job named is not deleted. Add one that answers the probe and exits non-zero. --- .github/actions/mongodb-binary/action.yml | 45 +++ .github/actions/mongodb-binary/prewarm.mjs | 225 ++++++++++++ .../actions/mongodb-binary/prewarm.test.mjs | 333 ++++++++++++++++++ .../workflows/agents-integration-tests.yml | 10 + .github/workflows/backend-review.yml | 52 +-- .github/workflows/change-quality.yml | 6 + .github/workflows/codegraph-e2e-votes.yml | 3 + .github/workflows/favorites-quality.yml | 3 + .github/workflows/lighthouse.yml | 3 + .github/workflows/playwright-bombadil.yml | 3 + .github/workflows/playwright-mock.yml | 9 + .github/workflows/static-checks.yml | 10 + .github/workflows/workspace-acceptance.yml | 4 + scripts/static-checks.mts | 9 +- 14 files changed, 674 insertions(+), 41 deletions(-) create mode 100644 .github/actions/mongodb-binary/action.yml create mode 100644 .github/actions/mongodb-binary/prewarm.mjs create mode 100644 .github/actions/mongodb-binary/prewarm.test.mjs diff --git a/.github/actions/mongodb-binary/action.yml b/.github/actions/mongodb-binary/action.yml new file mode 100644 index 0000000000..b387e67a1c --- /dev/null +++ b/.github/actions/mongodb-binary/action.yml @@ -0,0 +1,45 @@ +name: MongoDB memory-server binary +description: >- + Restores the mongodb-memory-server binary cache and makes sure the mongod build the installed + mongodb-memory-server asks for is on disk and runs, so no test downloads it from inside a hook. + Use after dependencies are installed and before anything starts MongoMemoryServer. + +inputs: + working-directory: + description: >- + Directory holding the package-lock.json and the installed node_modules the tests use, + relative to the workspace. Only needed when the repository is checked out to a subpath. + required: false + default: '.' + +runs: + using: composite + steps: + # restore-keys keeps the previous binary warm across lockfile churn; a genuinely new binary + # version downloads once, in the next step, and is re-saved. + - name: Restore MongoDB memory-server binaries + id: cache + uses: actions/cache/restore@v5 + with: + path: ~/.cache/mongodb-binaries + key: mongodb-binaries-${{ runner.os }}-${{ hashFiles(inputs.working-directory == '.' && 'package-lock.json' || format('{0}/package-lock.json', inputs.working-directory)) }} + restore-keys: | + mongodb-binaries-${{ runner.os }}- + + - name: Provision the mongod binary + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + MONGODB_BINARY_CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} + run: node "$GITHUB_ACTION_PATH/prewarm.mjs" + + # Saved here rather than in a post-job step: only a binary that was just proven to run is + # promoted into the cache every later job restores, and a failing test run still leaves it + # behind for the next one. + - name: Save MongoDB memory-server binaries + if: steps.cache.outputs.cache-hit != 'true' + continue-on-error: true + uses: actions/cache/save@v5 + with: + path: ~/.cache/mongodb-binaries + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/actions/mongodb-binary/prewarm.mjs b/.github/actions/mongodb-binary/prewarm.mjs new file mode 100644 index 0000000000..e818f3b445 --- /dev/null +++ b/.github/actions/mongodb-binary/prewarm.mjs @@ -0,0 +1,225 @@ +#!/usr/bin/env node +/** + * Puts the mongod build the installed mongodb-memory-server asks for into + * `~/.cache/mongodb-binaries` before any test starts, and proves it runs. + * + * Left to itself the library downloads the ~120 MB archive the first time a + * test calls `MongoMemoryServer.create()`: inside a `beforeAll` with a 15 s + * budget. On a slow mirror that hook times out while the download carries on, + * and whatever starts next spawns a file that is still being written + * (`spawn ETXTBSY`). + * + * The home cache is the one location every consumer resolves no matter where + * it runs. The copy `npm ci` leaves in the root `node_modules/.cache` is only + * seen from the repository root: the library derives that directory from + * INIT_CWD or the working directory, so suites run from `api`, + * `packages/api` or `packages/data-schemas` look in their own, find nothing + * and download. A binary found anywhere else is therefore copied into the + * shared directory rather than trusted. + * + * The version, platform and archive are the library's own answer + * (`MongoBinary.getPath()`), read from the working directory's install, so a + * dependency bump moves the binary with it. Exits non-zero when no runnable + * binary could be put in place. + */ +import os from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + rmSync, + statSync, +} from 'node:fs'; + +const HOME_CACHE = path.join(os.homedir(), '.cache', 'mongodb-binaries'); +const CHECKSUM = '.md5'; +const DOWNLOAD_ATTEMPTS = 2; + +/** + * Where the library looks for an existing binary: MONGOMS_DOWNLOAD_DIR, the + * home cache, `node_modules/.cache` and `./mongodb-binaries`. + */ +const LOOKUP_LOCATIONS = 4; + +/** + * The directory the library checks first from any working directory: the home + * cache, unless the job points MONGOMS_DOWNLOAD_DIR somewhere else for its tests too. + */ +const SHARED = process.env.MONGOMS_DOWNLOAD_DIR + ? path.resolve(process.env.MONGOMS_DOWNLOAD_DIR) + : HOME_CACHE; + +/** A job that names its own mongod owns that file: it is checked, never copied or replaced. */ +const SYSTEM_BINARY = process.env.MONGOMS_SYSTEM_BINARY; + +/** A workflow command ends at the first newline, so a multi-line message is folded into one. */ +const headline = (error) => + String(error?.message ?? error) + .split(/\s*\n\s*/) + .filter(Boolean) + .join(' '); + +function loadLibrary() { + const require = createRequire(path.join(process.cwd(), 'package.json')); + try { + return require('mongodb-memory-server'); + } catch (error) { + throw new Error( + `mongodb-memory-server is not installed under ${process.cwd()}; run this action after dependencies are installed (${headline(error)})`, + ); + } +} + +function entries(directory) { + try { + return readdirSync(directory); + } catch { + return []; + } +} + +/** Modification time per file, so a binary the library replaced is not mistaken for the cached one. */ +function snapshot(directory) { + return new Map( + entries(directory).map((name) => [name, statSync(path.join(directory, name)).mtimeMs]), + ); +} + +function remove(binary) { + rmSync(binary, { force: true }); + rmSync(binary + CHECKSUM, { force: true }); +} + +/** The checksum sidecar travels with the binary: the library discards a binary that lacks one. */ +function copy(source, target) { + mkdirSync(path.dirname(target), { recursive: true }); + copyFileSync(source, target); + chmodSync(target, 0o755); + if (existsSync(source + CHECKSUM)) { + copyFileSync(source + CHECKSUM, target + CHECKSUM); + } +} + +/** The first line of `mongod --version`; throws with the reason when the binary cannot run. */ +function version(binary) { + const result = spawnSync(binary, ['--version'], { encoding: 'utf8', timeout: 60_000 }); + if (result.status === 0) { + return result.stdout.split('\n', 1)[0].trim(); + } + const reason = + result.error?.message || + result.stderr?.trim() || + `exit ${result.status ?? result.signal ?? 'unknown'}`; + throw new Error(`${binary} does not run: ${reason}`); +} + +function describe(located, binary, cached) { + if (SYSTEM_BINARY) { + return 'MONGOMS_SYSTEM_BINARY'; + } + if (located !== binary) { + return `copied from ${located}`; + } + return cached.get(path.basename(binary)) === statSync(binary).mtimeMs + ? 'already present' + : 'downloaded'; +} + +/** + * Checks whatever the library resolves. A binary that does not run is deleted + * wherever it was found, so the next call resolves something else: another + * copy that was already on disk, or a fresh download. + */ +async function provision({ MongoBinary, DryMongoBinary }) { + const cached = snapshot(SHARED); + const located = await MongoBinary.getPath(); + const inPlace = Boolean(SYSTEM_BINARY) || path.dirname(located) === SHARED; + const binary = inPlace ? located : path.join(SHARED, path.basename(located)); + if (!inPlace) { + copy(located, binary); + } + const source = describe(located, binary, cached); + try { + return { binary, source, version: version(binary) }; + } catch (error) { + if (!SYSTEM_BINARY) { + remove(binary); + remove(located); + DryMongoBinary.binaryCache.clear(); + } + throw Object.assign(error, { preexisting: source !== 'downloaded' }); + } +} + +/** + * Copies that were already on disk and do not run are discarded one at a time + * without spending a download, so a broken file in one location cannot hide a + * good one, or the download, behind it. The download itself gets two attempts: + * a second failure is the mirror or the archive, not chance. + */ +async function provisionWithRecovery(library) { + let discarded = 0; + let downloads = 0; + for (;;) { + try { + return await provision(library); + } catch (error) { + if (error.preexisting) { + discarded += 1; + } else { + downloads += 1; + } + if (SYSTEM_BINARY || downloads >= DOWNLOAD_ATTEMPTS || discarded > LOOKUP_LOCATIONS) { + throw error; + } + const next = error.preexisting ? 'Discarded it; resolving again' : 'Retrying the download'; + console.log(`::warning::${headline(error)}. ${next}.`); + } + } +} + +/** A newer binary supersedes the ones an older lockfile's cache carried over. */ +function prune(binary) { + const keep = new Set([path.basename(binary), path.basename(binary) + CHECKSUM]); + return entries(SHARED).filter((name) => { + if (!name.startsWith('mongod-') || keep.has(name)) { + return false; + } + rmSync(path.join(SHARED, name), { force: true }); + return true; + }); +} + +async function main() { + const library = loadLibrary(); + const result = await provisionWithRecovery(library); + console.log(`mongod ready: ${result.binary}`); + console.log(` ${result.version} (${result.source})`); + + if (SYSTEM_BINARY || SHARED !== HOME_CACHE) { + console.log(` outside ${HOME_CACHE}, so the binary cache does not cover it`); + } + if (SYSTEM_BINARY || result.source === 'already present') { + return; + } + const stale = prune(result.binary); + if (stale.length > 0) { + console.log(` removed superseded binaries: ${stale.join(', ')}`); + } + if (process.env.MONGODB_BINARY_CACHE_HIT === 'true') { + console.log( + '::warning::The restored MongoDB binary cache had no runnable mongod for this install, and an exact cache hit is never re-saved: every job repeats this until the cache entry is deleted or package-lock.json changes.', + ); + } +} + +main().catch((error) => { + console.error(error); + console.error(`::error title=MongoDB binary::${headline(error)}`); + process.exit(1); +}); diff --git a/.github/actions/mongodb-binary/prewarm.test.mjs b/.github/actions/mongodb-binary/prewarm.test.mjs new file mode 100644 index 0000000000..faac1bc31d --- /dev/null +++ b/.github/actions/mongodb-binary/prewarm.test.mjs @@ -0,0 +1,333 @@ +import os from 'node:os'; +import http from 'node:http'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import { spawn, spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; + +/** + * Runs the real script against the installed mongodb-memory-server: its own + * resolver, lookup order, memoisation, download, archive checksum, extraction + * and checksum sidecar. Only the network is replaced, by pointing the library's + * MONGOMS_DOWNLOAD_MIRROR at a local server. It hands out an archive whose + * `bin/mongod` is a shell script, plus the `.md5` the library verifies that + * archive against, and records each archive request, so a test also knows how + * many downloads the script caused. + */ +const require = createRequire(import.meta.url); +const SCRIPT = fileURLToPath(new URL('./prewarm.mjs', import.meta.url)); +const LIBRARY = path.dirname(require.resolve('mongodb-memory-server/package.json')); +const posixOnly = { skip: process.platform === 'win32' && 'the stand-in mongod is a shell script' }; + +/** The library discards a binary under 1 MiB as a truncated extraction. */ +const PADDING = `\n# ${'-'.repeat(76)}`.repeat(14_000); +const GOOD = `#!/bin/sh\necho "db version v9.9.9"\necho "Build Info: {}"\nexit 0${PADDING}\n`; +const BROKEN = `#!/bin/sh\necho "cannot execute" >&2\nexit 3${PADDING}\n`; + +const SCRATCH = mkdtempSync(path.join(os.tmpdir(), 'mongodb-binary-archives-')); +after(() => rmSync(SCRATCH, { recursive: true, force: true })); + +function archive(label, content) { + const root = path.join(SCRATCH, label); + mkdirSync(path.join(root, 'mongodb-fixture', 'bin'), { recursive: true }); + writeFileSync(path.join(root, 'mongodb-fixture', 'bin', 'mongod'), content, { mode: 0o755 }); + const file = path.join(SCRATCH, `${label}.tgz`); + const packed = spawnSync('tar', ['-czf', file, '-C', root, 'mongodb-fixture']); + assert.equal(packed.status, 0, String(packed.stderr)); + return readFileSync(file); +} + +const ARCHIVES = posixOnly.skip + ? {} + : { good: archive('good', GOOD), broken: archive('broken', BROKEN) }; + +/** What the installed library calls the binary on this machine, e.g. `mongod-x64-ubuntu-8.2.6`. */ +async function binaryName() { + const { DryMongoBinary } = require('mongodb-memory-server'); + return DryMongoBinary.getBinaryName(await DryMongoBinary.generateOptions()); +} + +async function fixture(t, { serve = 'good', installed = true } = {}) { + const root = mkdtempSync(path.join(os.tmpdir(), 'mongodb-binary-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const home = path.join(root, 'home'); + const work = path.join(root, 'work'); + const homeCache = path.join(home, '.cache', 'mongodb-binaries'); + const modulesCache = path.join(work, 'node_modules', '.cache', 'mongodb-memory-server'); + const name = await binaryName(); + mkdirSync(home, { recursive: true }); + mkdirSync(path.join(work, 'node_modules'), { recursive: true }); + writeFileSync(path.join(work, 'package.json'), '{"name":"fixture","private":true}'); + if (installed) { + symlinkSync(LIBRARY, path.join(work, 'node_modules', 'mongodb-memory-server'), 'dir'); + } + + const downloads = []; + const server = http.createServer((request, response) => { + const packed = ARCHIVES[serve]; + const checksum = request.url.endsWith('.md5'); + if (!checksum) { + downloads.push(request.url); + } + if (!packed) { + response.writeHead(404).end(); + return; + } + const body = checksum + ? Buffer.from(`${createHash('md5').update(packed).digest('hex')} mongod.tgz\n`) + : packed; + response.writeHead(200, { 'content-length': body.length }); + response.end(body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + + const place = (directory, content = GOOD, file = name) => { + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, file), content); + chmodSync(path.join(directory, file), 0o755); + writeFileSync(path.join(directory, `${file}.md5`), `0 *${file}\n`); + return path.join(directory, file); + }; + + /** A child process, not spawnSync: the mirror above answers from this process's event loop. */ + const run = (env = {}) => + new Promise((resolve) => { + const inherited = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !key.startsWith('MONGOMS_') && key !== 'INIT_CWD', + ), + ); + const child = spawn(process.execPath, [SCRIPT], { + cwd: work, + env: { + ...inherited, + HOME: home, + MONGOMS_DOWNLOAD_MIRROR: `http://127.0.0.1:${server.address().port}`, + MONGOMS_USE_HTTP: 'true', + MONGOMS_MAX_RETRIES: '1', + ...env, + }, + }); + let output = ''; + child.stdout.on('data', (chunk) => (output += chunk)); + child.stderr.on('data', (chunk) => (output += chunk)); + child.on('close', (status) => resolve({ status, output })); + }); + + const cached = () => (existsSync(homeCache) ? readdirSync(homeCache).sort() : []); + return { root, name, homeCache, modulesCache, downloads, place, run, cached }; +} + +const firstLine = (file) => readFileSync(file, 'utf8').split('\n', 2).join('\n'); +const RUNS = '#!/bin/sh\necho "db version v9.9.9"'; + +test( + 'a binary already in the home cache is verified and nothing is downloaded', + posixOnly, + async (t) => { + const { homeCache, downloads, place, run, cached } = await fixture(t); + place(homeCache); + place(homeCache, GOOD, 'mongod-x64-ubuntu-7.0.14'); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.match(output, /db version v9\.9\.9 \(already present\)/); + assert.equal(downloads.length, 0); + assert.ok( + cached().includes('mongod-x64-ubuntu-7.0.14'), + 'an untouched cache is left as it was', + ); + }, +); + +test('a missing binary is downloaded once, before anything else needs it', posixOnly, async (t) => { + const { name, homeCache, downloads, run, cached } = await fixture(t); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.ok(output.includes(`mongod ready: ${path.join(homeCache, name)}`), output); + assert.match(output, /db version v9\.9\.9 \(downloaded\)/); + assert.equal(downloads.length, 1); + assert.deepEqual(cached(), [name, `${name}.md5`]); + + const again = await run(); + assert.equal(again.status, 0, again.output); + assert.match(again.output, /\(already present\)/); + assert.equal(downloads.length, 1, 'the second run finds what the first one left'); +}); + +test( + 'a binary only the repository root can see is copied into the home cache', + posixOnly, + async (t) => { + const { name, homeCache, modulesCache, downloads, place, run, cached } = await fixture(t); + const postinstall = place(modulesCache); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.match(output, /\(copied from .*node_modules\/\.cache\/mongodb-memory-server\//); + assert.equal(downloads.length, 0); + assert.deepEqual(cached(), [name, `${name}.md5`]); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + assert.ok(existsSync(postinstall), 'the original stays where npm ci put it'); + }, +); + +test( + 'a cached binary that does not run is replaced by one fresh download', + posixOnly, + async (t) => { + const { name, homeCache, downloads, place, run } = await fixture(t); + place(homeCache, BROKEN); + const { status, output } = await run({ MONGODB_BINARY_CACHE_HIT: 'true' }); + assert.equal(status, 0, output); + assert.match(output, /::warning::.*does not run: cannot execute\. Discarded it; resolving/); + assert.match(output, /db version v9\.9\.9 \(downloaded\)/); + assert.match(output, /::warning::The restored MongoDB binary cache had no runnable mongod/); + assert.equal(downloads.length, 1); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + }, +); + +test( + 'a broken binary outside the home cache is not copied back on the retry', + posixOnly, + async (t) => { + const { name, homeCache, modulesCache, downloads, place, run } = await fixture(t); + const postinstall = place(modulesCache, BROKEN); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.equal(downloads.length, 1); + assert.ok(!existsSync(postinstall)); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + }, +); + +test( + 'broken copies in both places are discarded without using up the download', + posixOnly, + async (t) => { + const { name, homeCache, modulesCache, downloads, place, run } = await fixture(t); + place(homeCache, BROKEN); + const postinstall = place(modulesCache, BROKEN); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.equal(output.match(/Discarded it; resolving again/g)?.length, 2, output); + assert.match(output, /db version v9\.9\.9 \(downloaded\)/); + assert.equal(downloads.length, 1); + assert.ok(!existsSync(postinstall)); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + }, +); + +test( + 'the job fails when the download keeps producing a binary that does not run', + posixOnly, + async (t) => { + const { downloads, run, cached } = await fixture(t, { serve: 'broken' }); + const { status, output } = await run(); + assert.equal(status, 1, output); + assert.match(output, /::error title=MongoDB binary::.*does not run: cannot execute/); + assert.equal(downloads.length, 2, 'two downloads, not a loop'); + assert.deepEqual( + cached().filter((file) => file.startsWith('mongod-')), + [], + 'nothing unusable is left behind to be cached', + ); + }, +); + +test( + 'the job fails with the whole reason on one line when the download fails', + posixOnly, + async (t) => { + const { downloads, run } = await fixture(t, { serve: 'nothing' }); + const { status, output } = await run(); + assert.equal(status, 1, output); + const annotation = output.split('\n').find((line) => line.startsWith('::error ')); + assert.match( + annotation, + /^::error title=MongoDB binary::Download failed for url "http:\/\/127\.0\.0\.1:\d+\//, + ); + assert.match(annotation, /404/, 'the status the library reports on its second line is kept'); + assert.equal(downloads.length, 2); + }, +); + +test('the job fails when dependencies are not installed', posixOnly, async (t) => { + const { downloads, run } = await fixture(t, { installed: false }); + const { status, output } = await run(); + assert.equal(status, 1, output); + assert.match( + output, + /::error title=MongoDB binary::mongodb-memory-server is not installed under /, + ); + assert.equal(downloads.length, 0); +}); + +test( + 'a new binary removes the versions an older cache carried over, and nothing else', + posixOnly, + async (t) => { + const { name, homeCache, place, run, cached } = await fixture(t); + place(homeCache, GOOD, 'mongod-x64-ubuntu-7.0.14'); + writeFileSync(path.join(homeCache, 'unrelated.txt'), 'keep'); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.match( + output, + /removed superseded binaries: mongod-x64-ubuntu-7\.0\.14, mongod-x64-ubuntu-7\.0\.14\.md5/, + ); + assert.deepEqual(cached(), [name, `${name}.md5`, 'unrelated.txt'].sort()); + }, +); + +test( + 'MONGOMS_DOWNLOAD_DIR names the shared directory instead of the home cache', + posixOnly, + async (t) => { + const { root, name, modulesCache, downloads, place, run, cached } = await fixture(t); + const directory = path.join(root, 'elsewhere'); + place(modulesCache); + const { status, output } = await run({ MONGOMS_DOWNLOAD_DIR: directory }); + assert.equal(status, 0, output); + assert.ok(output.includes(`mongod ready: ${path.join(directory, name)}`), output); + assert.match(output, /so the binary cache does not cover it/); + assert.equal(downloads.length, 0); + assert.deepEqual(readdirSync(directory).sort(), [name, `${name}.md5`]); + assert.deepEqual(cached(), []); + }, +); + +test('a system binary is checked where it is and never copied or deleted', posixOnly, async (t) => { + const { root, downloads, place, run, cached } = await fixture(t); + const working = place(path.join(root, 'system'), GOOD, 'mongod'); + const good = await run({ MONGOMS_SYSTEM_BINARY: working }); + assert.equal(good.status, 0, good.output); + assert.ok(good.output.includes(`mongod ready: ${working}`), good.output); + assert.match(good.output, /\(MONGOMS_SYSTEM_BINARY\)/); + + /** Rejected by the library's own version probe, then by this script's exit-status check. */ + const failures = { silent: BROKEN, 'exits-non-zero': GOOD.replace('exit 0', 'exit 3') }; + for (const [label, content] of Object.entries(failures)) { + const failing = place(path.join(root, label), content, 'mongod'); + const bad = await run({ MONGOMS_SYSTEM_BINARY: failing }); + assert.equal(bad.status, 1, bad.output); + assert.match(bad.output, /::error title=MongoDB binary::/); + assert.ok(existsSync(failing), `${label}: a file the job named is not ours to delete`); + } + assert.equal(downloads.length, 0); + assert.deepEqual(cached(), []); +}); diff --git a/.github/workflows/agents-integration-tests.yml b/.github/workflows/agents-integration-tests.yml index 5b214ae1a7..f878c4c052 100644 --- a/.github/workflows/agents-integration-tests.yml +++ b/.github/workflows/agents-integration-tests.yml @@ -25,6 +25,7 @@ on: - 'packages/data-schemas/src/**' - 'packages/data-schemas/package.json' - '.github/workflows/agents-integration-tests.yml' + - '.github/actions/mongodb-binary/**' - '!**.md' permissions: @@ -109,6 +110,15 @@ jobs: if: steps.cache-api.outputs.cache-hit != 'true' run: npm run build:api + - name: Test MongoDB binary provisioning + run: node --test .github/actions/mongodb-binary/prewarm.test.mjs + + # Every suite here runs in one Jest process with 15s hooks. A cold download of the + # ~122MB mongod inside the first hook timed it out on a slow mirror day, and the suites + # after it then spawned the half-written file (`spawn ETXTBSY`). + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Run integration tests working-directory: packages/api env: diff --git a/.github/workflows/backend-review.yml b/.github/workflows/backend-review.yml index f5f9a6a28a..bddbc4b9ef 100644 --- a/.github/workflows/backend-review.yml +++ b/.github/workflows/backend-review.yml @@ -481,15 +481,10 @@ jobs: # ~/.cache/mongodb-binaries on first use โ€” inside a 15s beforeAll hook, which is a # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- + # this download. The action restores the cached binary and, when it is missing, + # downloads it here, once, before Jest forks its workers. + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Run unit tests (shard ${{ matrix.shard }}/3) env: SELECTED: ${{ needs.codegraph_select.outputs.api_files }} @@ -567,19 +562,6 @@ jobs: name: build-data-provider path: packages/data-provider/dist - # mongodb-memory-server cold-downloads a ~122MB MongoDB binary into - # ~/.cache/mongodb-binaries on first use โ€” inside a 15s beforeAll hook, which is a - # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has - # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- - name: Run unit tests env: SELECTED: ${{ needs.codegraph_select.outputs.dataprovider_files }} @@ -662,15 +644,10 @@ jobs: # ~/.cache/mongodb-binaries on first use โ€” inside a 15s beforeAll hook, which is a # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- + # this download. The action restores the cached binary and, when it is missing, + # downloads it here, once, before Jest forks its workers. + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Run unit tests env: SELECTED: ${{ needs.codegraph_select.outputs.dataschemas_files }} @@ -767,15 +744,10 @@ jobs: # ~/.cache/mongodb-binaries on first use โ€” inside a 15s beforeAll hook, which is a # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- + # this download. The action restores the cached binary and, when it is missing, + # downloads it here, once, before Jest forks its workers. + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Run unit tests (shard ${{ matrix.shard }}/4) env: SELECTED: ${{ needs.codegraph_select.outputs.pkgapi_files }} diff --git a/.github/workflows/change-quality.yml b/.github/workflows/change-quality.yml index a1f3ef1c9a..a15505418e 100644 --- a/.github/workflows/change-quality.yml +++ b/.github/workflows/change-quality.yml @@ -129,6 +129,12 @@ jobs: fi env: QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }} + # The action comes from the workflow revision like the rest of the tooling; the binary + # is the one the measured source's own install asks for. + - name: Provision MongoDB memory-server binary + uses: ./tooling/.github/actions/mongodb-binary + with: + working-directory: source - name: Generate advisory report env: QUALITY_WORKSPACE: ${{ matrix.workspace }} diff --git a/.github/workflows/codegraph-e2e-votes.yml b/.github/workflows/codegraph-e2e-votes.yml index d1e40e0e9d..3c5228e02b 100644 --- a/.github/workflows/codegraph-e2e-votes.yml +++ b/.github/workflows/codegraph-e2e-votes.yml @@ -185,6 +185,9 @@ jobs: if: steps.cache-client-app.outputs.cache-hit != 'true' run: npm run build:client + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version diff --git a/.github/workflows/favorites-quality.yml b/.github/workflows/favorites-quality.yml index 37b7454485..3ca0679c9f 100644 --- a/.github/workflows/favorites-quality.yml +++ b/.github/workflows/favorites-quality.yml @@ -28,6 +28,7 @@ on: - 'package.json' - 'package-lock.json' - '.github/workflows/favorites-quality.yml' + - '.github/actions/mongodb-binary/**' permissions: contents: read @@ -73,6 +74,8 @@ jobs: --strict --types node scripts/crap.mts scripts/crap.test.mts - name: Verify the advisory reporter run: node --test scripts/crap.test.mts + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Contracts, compatibility, and CRAP report run: npm run test:quality:${{ matrix.suite }} --workspace packages/data-schemas - name: Mutate the real database implementation diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index f4f8032477..eeb6095385 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -11,6 +11,7 @@ on: - 'package.json' - 'package-lock.json' - '.github/workflows/lighthouse.yml' + - '.github/actions/mongodb-binary/**' - '!**.md' workflow_dispatch: @@ -76,6 +77,8 @@ jobs: packages/channel-gateway/node_modules api/node_modules key: ${{ steps.cache-node-modules.outputs.cache-primary-key }} + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Install lockfile-pinned Chromium run: npx playwright install --no-shell chromium # The audit keeps the lockfile-pinned Chromium build, so its download above stays fatal. diff --git a/.github/workflows/playwright-bombadil.yml b/.github/workflows/playwright-bombadil.yml index cf10b39163..295d51cde8 100644 --- a/.github/workflows/playwright-bombadil.yml +++ b/.github/workflows/playwright-bombadil.yml @@ -125,6 +125,9 @@ jobs: if: steps.cache-client-app.outputs.cache-hit != 'true' run: npm run build:client + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 56cb484d7f..8c0daa0681 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -307,6 +307,9 @@ jobs: if: steps.cache-client-app.outputs.cache-hit != 'true' run: npm run build:client + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version @@ -508,6 +511,9 @@ jobs: - name: Build e2e dependencies run: npm run e2e:prepare + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version @@ -655,6 +661,9 @@ jobs: if: steps.cache-node-modules.outputs.cache-hit != 'true' run: npm ci + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index f4a3795300..f45dd111bb 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -18,6 +18,7 @@ on: - 'eslint-suppressions.json' - '**/eslint-suppressions.json' - '.github/workflows/static-checks.yml' + - '.github/actions/mongodb-binary/**' - '.husky/pre-push' - '!**.md' @@ -123,6 +124,7 @@ jobs: - 'config/**' - 'packages/**' - '.github/workflows/static-checks.yml' + - '.github/actions/mongodb-binary/**' - '!**.md' i18n: - 'scripts/i18n*.mts' @@ -452,6 +454,12 @@ jobs: continue-on-error: true run: cp api/test/.env.test.example api/test/.env.test + - name: Provision MongoDB memory-server binary + id: config_mongodb + if: always() && steps.paths.outputs.config == 'true' + continue-on-error: true + uses: ./.github/actions/mongodb-binary + - name: Run config migration tests id: config_tests if: always() && steps.paths.outputs.config == 'true' @@ -720,6 +728,7 @@ jobs: CONFIG_API_OUTCOME: ${{ steps.config_api.outcome }} CONFIG_AUTH_OUTCOME: ${{ steps.config_auth.outcome }} CONFIG_ENV_OUTCOME: ${{ steps.config_env.outcome }} + CONFIG_MONGODB_OUTCOME: ${{ steps.config_mongodb.outcome }} CONFIG_TESTS_OUTCOME: ${{ steps.config_tests.outcome }} FIND_I18N_OUTCOME: ${{ steps.find_unused_i18n.outcome }} I18N_OUTCOME: ${{ steps.i18n.outcome }} @@ -761,6 +770,7 @@ jobs: record_failure "Config API build" "$CONFIG_API_OUTCOME" record_failure "Config auth preparation" "$CONFIG_AUTH_OUTCOME" record_failure "Config environment preparation" "$CONFIG_ENV_OUTCOME" + record_failure "Config MongoDB binary" "$CONFIG_MONGODB_OUTCOME" record_failure "Config migration tests" "$CONFIG_TESTS_OUTCOME" record_failure "Unused i18n scan" "$FIND_I18N_OUTCOME" record_failure "Unused i18n keys" "$I18N_OUTCOME" diff --git a/.github/workflows/workspace-acceptance.yml b/.github/workflows/workspace-acceptance.yml index 4aff1df3f4..c9cc20d1fc 100644 --- a/.github/workflows/workspace-acceptance.yml +++ b/.github/workflows/workspace-acceptance.yml @@ -6,6 +6,7 @@ on: paths: - 'e2e/byom/**' - '.github/workflows/workspace-acceptance.yml' + - '.github/actions/mongodb-binary/**' workflow_dispatch: permissions: @@ -59,6 +60,9 @@ jobs: npm run e2e:prepare npx playwright install chromium + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Build the real Code API and worker run: | npm ci --prefix .native-code/packages/code diff --git a/scripts/static-checks.mts b/scripts/static-checks.mts index 8061902fc3..1c176ebc9b 100644 --- a/scripts/static-checks.mts +++ b/scripts/static-checks.mts @@ -133,7 +133,14 @@ const FILTERS = { 'scripts/static-checks.mts', '.github/workflows/static-checks.yml', ], - config: ['api/**', 'config/**', 'packages/**', '.github/workflows/static-checks.yml', '!**.md'], + config: [ + 'api/**', + 'config/**', + 'packages/**', + '.github/workflows/static-checks.yml', + '.github/actions/mongodb-binary/**', + '!**.md', + ], i18n: [ 'scripts/i18n*.mts', 'scripts/static-checks.mts',