diff --git a/.github/actions/mongodb-binary/action.yml b/.github/actions/mongodb-binary/action.yml new file mode 100644 index 0000000000..b387e67a1c --- /dev/null +++ b/.github/actions/mongodb-binary/action.yml @@ -0,0 +1,45 @@ +name: MongoDB memory-server binary +description: >- + Restores the mongodb-memory-server binary cache and makes sure the mongod build the installed + mongodb-memory-server asks for is on disk and runs, so no test downloads it from inside a hook. + Use after dependencies are installed and before anything starts MongoMemoryServer. + +inputs: + working-directory: + description: >- + Directory holding the package-lock.json and the installed node_modules the tests use, + relative to the workspace. Only needed when the repository is checked out to a subpath. + required: false + default: '.' + +runs: + using: composite + steps: + # restore-keys keeps the previous binary warm across lockfile churn; a genuinely new binary + # version downloads once, in the next step, and is re-saved. + - name: Restore MongoDB memory-server binaries + id: cache + uses: actions/cache/restore@v5 + with: + path: ~/.cache/mongodb-binaries + key: mongodb-binaries-${{ runner.os }}-${{ hashFiles(inputs.working-directory == '.' && 'package-lock.json' || format('{0}/package-lock.json', inputs.working-directory)) }} + restore-keys: | + mongodb-binaries-${{ runner.os }}- + + - name: Provision the mongod binary + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + MONGODB_BINARY_CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} + run: node "$GITHUB_ACTION_PATH/prewarm.mjs" + + # Saved here rather than in a post-job step: only a binary that was just proven to run is + # promoted into the cache every later job restores, and a failing test run still leaves it + # behind for the next one. + - name: Save MongoDB memory-server binaries + if: steps.cache.outputs.cache-hit != 'true' + continue-on-error: true + uses: actions/cache/save@v5 + with: + path: ~/.cache/mongodb-binaries + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/actions/mongodb-binary/prewarm.mjs b/.github/actions/mongodb-binary/prewarm.mjs new file mode 100644 index 0000000000..e818f3b445 --- /dev/null +++ b/.github/actions/mongodb-binary/prewarm.mjs @@ -0,0 +1,225 @@ +#!/usr/bin/env node +/** + * Puts the mongod build the installed mongodb-memory-server asks for into + * `~/.cache/mongodb-binaries` before any test starts, and proves it runs. + * + * Left to itself the library downloads the ~120 MB archive the first time a + * test calls `MongoMemoryServer.create()`: inside a `beforeAll` with a 15 s + * budget. On a slow mirror that hook times out while the download carries on, + * and whatever starts next spawns a file that is still being written + * (`spawn ETXTBSY`). + * + * The home cache is the one location every consumer resolves no matter where + * it runs. The copy `npm ci` leaves in the root `node_modules/.cache` is only + * seen from the repository root: the library derives that directory from + * INIT_CWD or the working directory, so suites run from `api`, + * `packages/api` or `packages/data-schemas` look in their own, find nothing + * and download. A binary found anywhere else is therefore copied into the + * shared directory rather than trusted. + * + * The version, platform and archive are the library's own answer + * (`MongoBinary.getPath()`), read from the working directory's install, so a + * dependency bump moves the binary with it. Exits non-zero when no runnable + * binary could be put in place. + */ +import os from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + rmSync, + statSync, +} from 'node:fs'; + +const HOME_CACHE = path.join(os.homedir(), '.cache', 'mongodb-binaries'); +const CHECKSUM = '.md5'; +const DOWNLOAD_ATTEMPTS = 2; + +/** + * Where the library looks for an existing binary: MONGOMS_DOWNLOAD_DIR, the + * home cache, `node_modules/.cache` and `./mongodb-binaries`. + */ +const LOOKUP_LOCATIONS = 4; + +/** + * The directory the library checks first from any working directory: the home + * cache, unless the job points MONGOMS_DOWNLOAD_DIR somewhere else for its tests too. + */ +const SHARED = process.env.MONGOMS_DOWNLOAD_DIR + ? path.resolve(process.env.MONGOMS_DOWNLOAD_DIR) + : HOME_CACHE; + +/** A job that names its own mongod owns that file: it is checked, never copied or replaced. */ +const SYSTEM_BINARY = process.env.MONGOMS_SYSTEM_BINARY; + +/** A workflow command ends at the first newline, so a multi-line message is folded into one. */ +const headline = (error) => + String(error?.message ?? error) + .split(/\s*\n\s*/) + .filter(Boolean) + .join(' '); + +function loadLibrary() { + const require = createRequire(path.join(process.cwd(), 'package.json')); + try { + return require('mongodb-memory-server'); + } catch (error) { + throw new Error( + `mongodb-memory-server is not installed under ${process.cwd()}; run this action after dependencies are installed (${headline(error)})`, + ); + } +} + +function entries(directory) { + try { + return readdirSync(directory); + } catch { + return []; + } +} + +/** Modification time per file, so a binary the library replaced is not mistaken for the cached one. */ +function snapshot(directory) { + return new Map( + entries(directory).map((name) => [name, statSync(path.join(directory, name)).mtimeMs]), + ); +} + +function remove(binary) { + rmSync(binary, { force: true }); + rmSync(binary + CHECKSUM, { force: true }); +} + +/** The checksum sidecar travels with the binary: the library discards a binary that lacks one. */ +function copy(source, target) { + mkdirSync(path.dirname(target), { recursive: true }); + copyFileSync(source, target); + chmodSync(target, 0o755); + if (existsSync(source + CHECKSUM)) { + copyFileSync(source + CHECKSUM, target + CHECKSUM); + } +} + +/** The first line of `mongod --version`; throws with the reason when the binary cannot run. */ +function version(binary) { + const result = spawnSync(binary, ['--version'], { encoding: 'utf8', timeout: 60_000 }); + if (result.status === 0) { + return result.stdout.split('\n', 1)[0].trim(); + } + const reason = + result.error?.message || + result.stderr?.trim() || + `exit ${result.status ?? result.signal ?? 'unknown'}`; + throw new Error(`${binary} does not run: ${reason}`); +} + +function describe(located, binary, cached) { + if (SYSTEM_BINARY) { + return 'MONGOMS_SYSTEM_BINARY'; + } + if (located !== binary) { + return `copied from ${located}`; + } + return cached.get(path.basename(binary)) === statSync(binary).mtimeMs + ? 'already present' + : 'downloaded'; +} + +/** + * Checks whatever the library resolves. A binary that does not run is deleted + * wherever it was found, so the next call resolves something else: another + * copy that was already on disk, or a fresh download. + */ +async function provision({ MongoBinary, DryMongoBinary }) { + const cached = snapshot(SHARED); + const located = await MongoBinary.getPath(); + const inPlace = Boolean(SYSTEM_BINARY) || path.dirname(located) === SHARED; + const binary = inPlace ? located : path.join(SHARED, path.basename(located)); + if (!inPlace) { + copy(located, binary); + } + const source = describe(located, binary, cached); + try { + return { binary, source, version: version(binary) }; + } catch (error) { + if (!SYSTEM_BINARY) { + remove(binary); + remove(located); + DryMongoBinary.binaryCache.clear(); + } + throw Object.assign(error, { preexisting: source !== 'downloaded' }); + } +} + +/** + * Copies that were already on disk and do not run are discarded one at a time + * without spending a download, so a broken file in one location cannot hide a + * good one, or the download, behind it. The download itself gets two attempts: + * a second failure is the mirror or the archive, not chance. + */ +async function provisionWithRecovery(library) { + let discarded = 0; + let downloads = 0; + for (;;) { + try { + return await provision(library); + } catch (error) { + if (error.preexisting) { + discarded += 1; + } else { + downloads += 1; + } + if (SYSTEM_BINARY || downloads >= DOWNLOAD_ATTEMPTS || discarded > LOOKUP_LOCATIONS) { + throw error; + } + const next = error.preexisting ? 'Discarded it; resolving again' : 'Retrying the download'; + console.log(`::warning::${headline(error)}. ${next}.`); + } + } +} + +/** A newer binary supersedes the ones an older lockfile's cache carried over. */ +function prune(binary) { + const keep = new Set([path.basename(binary), path.basename(binary) + CHECKSUM]); + return entries(SHARED).filter((name) => { + if (!name.startsWith('mongod-') || keep.has(name)) { + return false; + } + rmSync(path.join(SHARED, name), { force: true }); + return true; + }); +} + +async function main() { + const library = loadLibrary(); + const result = await provisionWithRecovery(library); + console.log(`mongod ready: ${result.binary}`); + console.log(` ${result.version} (${result.source})`); + + if (SYSTEM_BINARY || SHARED !== HOME_CACHE) { + console.log(` outside ${HOME_CACHE}, so the binary cache does not cover it`); + } + if (SYSTEM_BINARY || result.source === 'already present') { + return; + } + const stale = prune(result.binary); + if (stale.length > 0) { + console.log(` removed superseded binaries: ${stale.join(', ')}`); + } + if (process.env.MONGODB_BINARY_CACHE_HIT === 'true') { + console.log( + '::warning::The restored MongoDB binary cache had no runnable mongod for this install, and an exact cache hit is never re-saved: every job repeats this until the cache entry is deleted or package-lock.json changes.', + ); + } +} + +main().catch((error) => { + console.error(error); + console.error(`::error title=MongoDB binary::${headline(error)}`); + process.exit(1); +}); diff --git a/.github/actions/mongodb-binary/prewarm.test.mjs b/.github/actions/mongodb-binary/prewarm.test.mjs new file mode 100644 index 0000000000..faac1bc31d --- /dev/null +++ b/.github/actions/mongodb-binary/prewarm.test.mjs @@ -0,0 +1,333 @@ +import os from 'node:os'; +import http from 'node:http'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import { spawn, spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; + +/** + * Runs the real script against the installed mongodb-memory-server: its own + * resolver, lookup order, memoisation, download, archive checksum, extraction + * and checksum sidecar. Only the network is replaced, by pointing the library's + * MONGOMS_DOWNLOAD_MIRROR at a local server. It hands out an archive whose + * `bin/mongod` is a shell script, plus the `.md5` the library verifies that + * archive against, and records each archive request, so a test also knows how + * many downloads the script caused. + */ +const require = createRequire(import.meta.url); +const SCRIPT = fileURLToPath(new URL('./prewarm.mjs', import.meta.url)); +const LIBRARY = path.dirname(require.resolve('mongodb-memory-server/package.json')); +const posixOnly = { skip: process.platform === 'win32' && 'the stand-in mongod is a shell script' }; + +/** The library discards a binary under 1 MiB as a truncated extraction. */ +const PADDING = `\n# ${'-'.repeat(76)}`.repeat(14_000); +const GOOD = `#!/bin/sh\necho "db version v9.9.9"\necho "Build Info: {}"\nexit 0${PADDING}\n`; +const BROKEN = `#!/bin/sh\necho "cannot execute" >&2\nexit 3${PADDING}\n`; + +const SCRATCH = mkdtempSync(path.join(os.tmpdir(), 'mongodb-binary-archives-')); +after(() => rmSync(SCRATCH, { recursive: true, force: true })); + +function archive(label, content) { + const root = path.join(SCRATCH, label); + mkdirSync(path.join(root, 'mongodb-fixture', 'bin'), { recursive: true }); + writeFileSync(path.join(root, 'mongodb-fixture', 'bin', 'mongod'), content, { mode: 0o755 }); + const file = path.join(SCRATCH, `${label}.tgz`); + const packed = spawnSync('tar', ['-czf', file, '-C', root, 'mongodb-fixture']); + assert.equal(packed.status, 0, String(packed.stderr)); + return readFileSync(file); +} + +const ARCHIVES = posixOnly.skip + ? {} + : { good: archive('good', GOOD), broken: archive('broken', BROKEN) }; + +/** What the installed library calls the binary on this machine, e.g. `mongod-x64-ubuntu-8.2.6`. */ +async function binaryName() { + const { DryMongoBinary } = require('mongodb-memory-server'); + return DryMongoBinary.getBinaryName(await DryMongoBinary.generateOptions()); +} + +async function fixture(t, { serve = 'good', installed = true } = {}) { + const root = mkdtempSync(path.join(os.tmpdir(), 'mongodb-binary-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const home = path.join(root, 'home'); + const work = path.join(root, 'work'); + const homeCache = path.join(home, '.cache', 'mongodb-binaries'); + const modulesCache = path.join(work, 'node_modules', '.cache', 'mongodb-memory-server'); + const name = await binaryName(); + mkdirSync(home, { recursive: true }); + mkdirSync(path.join(work, 'node_modules'), { recursive: true }); + writeFileSync(path.join(work, 'package.json'), '{"name":"fixture","private":true}'); + if (installed) { + symlinkSync(LIBRARY, path.join(work, 'node_modules', 'mongodb-memory-server'), 'dir'); + } + + const downloads = []; + const server = http.createServer((request, response) => { + const packed = ARCHIVES[serve]; + const checksum = request.url.endsWith('.md5'); + if (!checksum) { + downloads.push(request.url); + } + if (!packed) { + response.writeHead(404).end(); + return; + } + const body = checksum + ? Buffer.from(`${createHash('md5').update(packed).digest('hex')} mongod.tgz\n`) + : packed; + response.writeHead(200, { 'content-length': body.length }); + response.end(body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + + const place = (directory, content = GOOD, file = name) => { + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, file), content); + chmodSync(path.join(directory, file), 0o755); + writeFileSync(path.join(directory, `${file}.md5`), `0 *${file}\n`); + return path.join(directory, file); + }; + + /** A child process, not spawnSync: the mirror above answers from this process's event loop. */ + const run = (env = {}) => + new Promise((resolve) => { + const inherited = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !key.startsWith('MONGOMS_') && key !== 'INIT_CWD', + ), + ); + const child = spawn(process.execPath, [SCRIPT], { + cwd: work, + env: { + ...inherited, + HOME: home, + MONGOMS_DOWNLOAD_MIRROR: `http://127.0.0.1:${server.address().port}`, + MONGOMS_USE_HTTP: 'true', + MONGOMS_MAX_RETRIES: '1', + ...env, + }, + }); + let output = ''; + child.stdout.on('data', (chunk) => (output += chunk)); + child.stderr.on('data', (chunk) => (output += chunk)); + child.on('close', (status) => resolve({ status, output })); + }); + + const cached = () => (existsSync(homeCache) ? readdirSync(homeCache).sort() : []); + return { root, name, homeCache, modulesCache, downloads, place, run, cached }; +} + +const firstLine = (file) => readFileSync(file, 'utf8').split('\n', 2).join('\n'); +const RUNS = '#!/bin/sh\necho "db version v9.9.9"'; + +test( + 'a binary already in the home cache is verified and nothing is downloaded', + posixOnly, + async (t) => { + const { homeCache, downloads, place, run, cached } = await fixture(t); + place(homeCache); + place(homeCache, GOOD, 'mongod-x64-ubuntu-7.0.14'); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.match(output, /db version v9\.9\.9 \(already present\)/); + assert.equal(downloads.length, 0); + assert.ok( + cached().includes('mongod-x64-ubuntu-7.0.14'), + 'an untouched cache is left as it was', + ); + }, +); + +test('a missing binary is downloaded once, before anything else needs it', posixOnly, async (t) => { + const { name, homeCache, downloads, run, cached } = await fixture(t); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.ok(output.includes(`mongod ready: ${path.join(homeCache, name)}`), output); + assert.match(output, /db version v9\.9\.9 \(downloaded\)/); + assert.equal(downloads.length, 1); + assert.deepEqual(cached(), [name, `${name}.md5`]); + + const again = await run(); + assert.equal(again.status, 0, again.output); + assert.match(again.output, /\(already present\)/); + assert.equal(downloads.length, 1, 'the second run finds what the first one left'); +}); + +test( + 'a binary only the repository root can see is copied into the home cache', + posixOnly, + async (t) => { + const { name, homeCache, modulesCache, downloads, place, run, cached } = await fixture(t); + const postinstall = place(modulesCache); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.match(output, /\(copied from .*node_modules\/\.cache\/mongodb-memory-server\//); + assert.equal(downloads.length, 0); + assert.deepEqual(cached(), [name, `${name}.md5`]); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + assert.ok(existsSync(postinstall), 'the original stays where npm ci put it'); + }, +); + +test( + 'a cached binary that does not run is replaced by one fresh download', + posixOnly, + async (t) => { + const { name, homeCache, downloads, place, run } = await fixture(t); + place(homeCache, BROKEN); + const { status, output } = await run({ MONGODB_BINARY_CACHE_HIT: 'true' }); + assert.equal(status, 0, output); + assert.match(output, /::warning::.*does not run: cannot execute\. Discarded it; resolving/); + assert.match(output, /db version v9\.9\.9 \(downloaded\)/); + assert.match(output, /::warning::The restored MongoDB binary cache had no runnable mongod/); + assert.equal(downloads.length, 1); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + }, +); + +test( + 'a broken binary outside the home cache is not copied back on the retry', + posixOnly, + async (t) => { + const { name, homeCache, modulesCache, downloads, place, run } = await fixture(t); + const postinstall = place(modulesCache, BROKEN); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.equal(downloads.length, 1); + assert.ok(!existsSync(postinstall)); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + }, +); + +test( + 'broken copies in both places are discarded without using up the download', + posixOnly, + async (t) => { + const { name, homeCache, modulesCache, downloads, place, run } = await fixture(t); + place(homeCache, BROKEN); + const postinstall = place(modulesCache, BROKEN); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.equal(output.match(/Discarded it; resolving again/g)?.length, 2, output); + assert.match(output, /db version v9\.9\.9 \(downloaded\)/); + assert.equal(downloads.length, 1); + assert.ok(!existsSync(postinstall)); + assert.equal(firstLine(path.join(homeCache, name)), RUNS); + }, +); + +test( + 'the job fails when the download keeps producing a binary that does not run', + posixOnly, + async (t) => { + const { downloads, run, cached } = await fixture(t, { serve: 'broken' }); + const { status, output } = await run(); + assert.equal(status, 1, output); + assert.match(output, /::error title=MongoDB binary::.*does not run: cannot execute/); + assert.equal(downloads.length, 2, 'two downloads, not a loop'); + assert.deepEqual( + cached().filter((file) => file.startsWith('mongod-')), + [], + 'nothing unusable is left behind to be cached', + ); + }, +); + +test( + 'the job fails with the whole reason on one line when the download fails', + posixOnly, + async (t) => { + const { downloads, run } = await fixture(t, { serve: 'nothing' }); + const { status, output } = await run(); + assert.equal(status, 1, output); + const annotation = output.split('\n').find((line) => line.startsWith('::error ')); + assert.match( + annotation, + /^::error title=MongoDB binary::Download failed for url "http:\/\/127\.0\.0\.1:\d+\//, + ); + assert.match(annotation, /404/, 'the status the library reports on its second line is kept'); + assert.equal(downloads.length, 2); + }, +); + +test('the job fails when dependencies are not installed', posixOnly, async (t) => { + const { downloads, run } = await fixture(t, { installed: false }); + const { status, output } = await run(); + assert.equal(status, 1, output); + assert.match( + output, + /::error title=MongoDB binary::mongodb-memory-server is not installed under /, + ); + assert.equal(downloads.length, 0); +}); + +test( + 'a new binary removes the versions an older cache carried over, and nothing else', + posixOnly, + async (t) => { + const { name, homeCache, place, run, cached } = await fixture(t); + place(homeCache, GOOD, 'mongod-x64-ubuntu-7.0.14'); + writeFileSync(path.join(homeCache, 'unrelated.txt'), 'keep'); + const { status, output } = await run(); + assert.equal(status, 0, output); + assert.match( + output, + /removed superseded binaries: mongod-x64-ubuntu-7\.0\.14, mongod-x64-ubuntu-7\.0\.14\.md5/, + ); + assert.deepEqual(cached(), [name, `${name}.md5`, 'unrelated.txt'].sort()); + }, +); + +test( + 'MONGOMS_DOWNLOAD_DIR names the shared directory instead of the home cache', + posixOnly, + async (t) => { + const { root, name, modulesCache, downloads, place, run, cached } = await fixture(t); + const directory = path.join(root, 'elsewhere'); + place(modulesCache); + const { status, output } = await run({ MONGOMS_DOWNLOAD_DIR: directory }); + assert.equal(status, 0, output); + assert.ok(output.includes(`mongod ready: ${path.join(directory, name)}`), output); + assert.match(output, /so the binary cache does not cover it/); + assert.equal(downloads.length, 0); + assert.deepEqual(readdirSync(directory).sort(), [name, `${name}.md5`]); + assert.deepEqual(cached(), []); + }, +); + +test('a system binary is checked where it is and never copied or deleted', posixOnly, async (t) => { + const { root, downloads, place, run, cached } = await fixture(t); + const working = place(path.join(root, 'system'), GOOD, 'mongod'); + const good = await run({ MONGOMS_SYSTEM_BINARY: working }); + assert.equal(good.status, 0, good.output); + assert.ok(good.output.includes(`mongod ready: ${working}`), good.output); + assert.match(good.output, /\(MONGOMS_SYSTEM_BINARY\)/); + + /** Rejected by the library's own version probe, then by this script's exit-status check. */ + const failures = { silent: BROKEN, 'exits-non-zero': GOOD.replace('exit 0', 'exit 3') }; + for (const [label, content] of Object.entries(failures)) { + const failing = place(path.join(root, label), content, 'mongod'); + const bad = await run({ MONGOMS_SYSTEM_BINARY: failing }); + assert.equal(bad.status, 1, bad.output); + assert.match(bad.output, /::error title=MongoDB binary::/); + assert.ok(existsSync(failing), `${label}: a file the job named is not ours to delete`); + } + assert.equal(downloads.length, 0); + assert.deepEqual(cached(), []); +}); diff --git a/.github/workflows/agents-integration-tests.yml b/.github/workflows/agents-integration-tests.yml index 5b214ae1a7..f878c4c052 100644 --- a/.github/workflows/agents-integration-tests.yml +++ b/.github/workflows/agents-integration-tests.yml @@ -25,6 +25,7 @@ on: - 'packages/data-schemas/src/**' - 'packages/data-schemas/package.json' - '.github/workflows/agents-integration-tests.yml' + - '.github/actions/mongodb-binary/**' - '!**.md' permissions: @@ -109,6 +110,15 @@ jobs: if: steps.cache-api.outputs.cache-hit != 'true' run: npm run build:api + - name: Test MongoDB binary provisioning + run: node --test .github/actions/mongodb-binary/prewarm.test.mjs + + # Every suite here runs in one Jest process with 15s hooks. A cold download of the + # ~122MB mongod inside the first hook timed it out on a slow mirror day, and the suites + # after it then spawned the half-written file (`spawn ETXTBSY`). + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Run integration tests working-directory: packages/api env: diff --git a/.github/workflows/backend-review.yml b/.github/workflows/backend-review.yml index f5f9a6a28a..bddbc4b9ef 100644 --- a/.github/workflows/backend-review.yml +++ b/.github/workflows/backend-review.yml @@ -481,15 +481,10 @@ jobs: # ~/.cache/mongodb-binaries on first use — inside a 15s beforeAll hook, which is a # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- + # this download. The action restores the cached binary and, when it is missing, + # downloads it here, once, before Jest forks its workers. + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Run unit tests (shard ${{ matrix.shard }}/3) env: SELECTED: ${{ needs.codegraph_select.outputs.api_files }} @@ -567,19 +562,6 @@ jobs: name: build-data-provider path: packages/data-provider/dist - # mongodb-memory-server cold-downloads a ~122MB MongoDB binary into - # ~/.cache/mongodb-binaries on first use — inside a 15s beforeAll hook, which is a - # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has - # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- - name: Run unit tests env: SELECTED: ${{ needs.codegraph_select.outputs.dataprovider_files }} @@ -662,15 +644,10 @@ jobs: # ~/.cache/mongodb-binaries on first use — inside a 15s beforeAll hook, which is a # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- + # this download. The action restores the cached binary and, when it is missing, + # downloads it here, once, before Jest forks its workers. + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Run unit tests env: SELECTED: ${{ needs.codegraph_select.outputs.dataschemas_files }} @@ -767,15 +744,10 @@ jobs: # ~/.cache/mongodb-binaries on first use — inside a 15s beforeAll hook, which is a # timeout on a slow mirror day. Every MISSED verdict the codegraph shadow evaluator has # ever recorded (9 across 6 PRs) plus several chronically flaky suites trace to exactly - # this download. restore-keys keeps the previous binary warm across lockfile churn; a - # genuinely new binary version downloads once and re-saves. - - name: Cache MongoDB memory-server binaries - uses: actions/cache@v5 - with: - path: ~/.cache/mongodb-binaries - key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - mongodb-binaries-${{ runner.os }}- + # this download. The action restores the cached binary and, when it is missing, + # downloads it here, once, before Jest forks its workers. + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Run unit tests (shard ${{ matrix.shard }}/4) env: SELECTED: ${{ needs.codegraph_select.outputs.pkgapi_files }} diff --git a/.github/workflows/change-quality.yml b/.github/workflows/change-quality.yml index a1f3ef1c9a..a15505418e 100644 --- a/.github/workflows/change-quality.yml +++ b/.github/workflows/change-quality.yml @@ -129,6 +129,12 @@ jobs: fi env: QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }} + # The action comes from the workflow revision like the rest of the tooling; the binary + # is the one the measured source's own install asks for. + - name: Provision MongoDB memory-server binary + uses: ./tooling/.github/actions/mongodb-binary + with: + working-directory: source - name: Generate advisory report env: QUALITY_WORKSPACE: ${{ matrix.workspace }} diff --git a/.github/workflows/codegraph-e2e-votes.yml b/.github/workflows/codegraph-e2e-votes.yml index d1e40e0e9d..3c5228e02b 100644 --- a/.github/workflows/codegraph-e2e-votes.yml +++ b/.github/workflows/codegraph-e2e-votes.yml @@ -185,6 +185,9 @@ jobs: if: steps.cache-client-app.outputs.cache-hit != 'true' run: npm run build:client + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version diff --git a/.github/workflows/favorites-quality.yml b/.github/workflows/favorites-quality.yml index 37b7454485..3ca0679c9f 100644 --- a/.github/workflows/favorites-quality.yml +++ b/.github/workflows/favorites-quality.yml @@ -28,6 +28,7 @@ on: - 'package.json' - 'package-lock.json' - '.github/workflows/favorites-quality.yml' + - '.github/actions/mongodb-binary/**' permissions: contents: read @@ -73,6 +74,8 @@ jobs: --strict --types node scripts/crap.mts scripts/crap.test.mts - name: Verify the advisory reporter run: node --test scripts/crap.test.mts + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Contracts, compatibility, and CRAP report run: npm run test:quality:${{ matrix.suite }} --workspace packages/data-schemas - name: Mutate the real database implementation diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index f4f8032477..eeb6095385 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -11,6 +11,7 @@ on: - 'package.json' - 'package-lock.json' - '.github/workflows/lighthouse.yml' + - '.github/actions/mongodb-binary/**' - '!**.md' workflow_dispatch: @@ -76,6 +77,8 @@ jobs: packages/channel-gateway/node_modules api/node_modules key: ${{ steps.cache-node-modules.outputs.cache-primary-key }} + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary - name: Install lockfile-pinned Chromium run: npx playwright install --no-shell chromium # The audit keeps the lockfile-pinned Chromium build, so its download above stays fatal. diff --git a/.github/workflows/playwright-bombadil.yml b/.github/workflows/playwright-bombadil.yml index cf10b39163..295d51cde8 100644 --- a/.github/workflows/playwright-bombadil.yml +++ b/.github/workflows/playwright-bombadil.yml @@ -125,6 +125,9 @@ jobs: if: steps.cache-client-app.outputs.cache-hit != 'true' run: npm run build:client + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml index 56cb484d7f..8c0daa0681 100644 --- a/.github/workflows/playwright-mock.yml +++ b/.github/workflows/playwright-mock.yml @@ -307,6 +307,9 @@ jobs: if: steps.cache-client-app.outputs.cache-hit != 'true' run: npm run build:client + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version @@ -508,6 +511,9 @@ jobs: - name: Build e2e dependencies run: npm run e2e:prepare + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version @@ -655,6 +661,9 @@ jobs: if: steps.cache-node-modules.outputs.cache-hit != 'true' run: npm ci + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Verify Chrome is present run: google-chrome --version diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index f4a3795300..f45dd111bb 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -18,6 +18,7 @@ on: - 'eslint-suppressions.json' - '**/eslint-suppressions.json' - '.github/workflows/static-checks.yml' + - '.github/actions/mongodb-binary/**' - '.husky/pre-push' - '!**.md' @@ -123,6 +124,7 @@ jobs: - 'config/**' - 'packages/**' - '.github/workflows/static-checks.yml' + - '.github/actions/mongodb-binary/**' - '!**.md' i18n: - 'scripts/i18n*.mts' @@ -452,6 +454,12 @@ jobs: continue-on-error: true run: cp api/test/.env.test.example api/test/.env.test + - name: Provision MongoDB memory-server binary + id: config_mongodb + if: always() && steps.paths.outputs.config == 'true' + continue-on-error: true + uses: ./.github/actions/mongodb-binary + - name: Run config migration tests id: config_tests if: always() && steps.paths.outputs.config == 'true' @@ -720,6 +728,7 @@ jobs: CONFIG_API_OUTCOME: ${{ steps.config_api.outcome }} CONFIG_AUTH_OUTCOME: ${{ steps.config_auth.outcome }} CONFIG_ENV_OUTCOME: ${{ steps.config_env.outcome }} + CONFIG_MONGODB_OUTCOME: ${{ steps.config_mongodb.outcome }} CONFIG_TESTS_OUTCOME: ${{ steps.config_tests.outcome }} FIND_I18N_OUTCOME: ${{ steps.find_unused_i18n.outcome }} I18N_OUTCOME: ${{ steps.i18n.outcome }} @@ -761,6 +770,7 @@ jobs: record_failure "Config API build" "$CONFIG_API_OUTCOME" record_failure "Config auth preparation" "$CONFIG_AUTH_OUTCOME" record_failure "Config environment preparation" "$CONFIG_ENV_OUTCOME" + record_failure "Config MongoDB binary" "$CONFIG_MONGODB_OUTCOME" record_failure "Config migration tests" "$CONFIG_TESTS_OUTCOME" record_failure "Unused i18n scan" "$FIND_I18N_OUTCOME" record_failure "Unused i18n keys" "$I18N_OUTCOME" diff --git a/.github/workflows/workspace-acceptance.yml b/.github/workflows/workspace-acceptance.yml index 4aff1df3f4..c9cc20d1fc 100644 --- a/.github/workflows/workspace-acceptance.yml +++ b/.github/workflows/workspace-acceptance.yml @@ -6,6 +6,7 @@ on: paths: - 'e2e/byom/**' - '.github/workflows/workspace-acceptance.yml' + - '.github/actions/mongodb-binary/**' workflow_dispatch: permissions: @@ -59,6 +60,9 @@ jobs: npm run e2e:prepare npx playwright install chromium + - name: Provision MongoDB memory-server binary + uses: ./.github/actions/mongodb-binary + - name: Build the real Code API and worker run: | npm ci --prefix .native-code/packages/code diff --git a/scripts/static-checks.mts b/scripts/static-checks.mts index 8061902fc3..1c176ebc9b 100644 --- a/scripts/static-checks.mts +++ b/scripts/static-checks.mts @@ -133,7 +133,14 @@ const FILTERS = { 'scripts/static-checks.mts', '.github/workflows/static-checks.yml', ], - config: ['api/**', 'config/**', 'packages/**', '.github/workflows/static-checks.yml', '!**.md'], + config: [ + 'api/**', + 'config/**', + 'packages/**', + '.github/workflows/static-checks.yml', + '.github/actions/mongodb-binary/**', + '!**.md', + ], i18n: [ 'scripts/i18n*.mts', 'scripts/static-checks.mts',