diff --git a/.github/workflows/agents-integration-tests.yml b/.github/workflows/agents-integration-tests.yml new file mode 100644 index 0000000000..8e841daa4c --- /dev/null +++ b/.github/workflows/agents-integration-tests.yml @@ -0,0 +1,90 @@ +name: Agents Integration Tests + +# Runs the packages/api `src/agents/**` integration specs (e.g. the durable HITL +# checkpointer against a real in-process MongoDB via mongodb-memory-server). These +# are `*.integration.spec.ts`, which `test:ci` deliberately excludes — without this +# job they run nowhere and their regressions guard nothing. +on: + pull_request: + branches: + - main + - dev + - dev-staging + - release/* + paths: + - 'packages/api/src/agents/**' + - 'packages/api/package.json' + - '.github/workflows/agents-integration-tests.yml' + +permissions: + contents: read + +jobs: + agents_integration_tests: + name: Integration Tests that use in-process MongoDB + timeout-minutes: 20 + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Use Node.js 24.16.0 + uses: actions/setup-node@v4 + with: + node-version: '24.16.0' + + - name: Restore node_modules cache + id: cache-node-modules + uses: actions/cache@v4 + with: + path: | + node_modules + api/node_modules + packages/api/node_modules + packages/data-provider/node_modules + packages/data-schemas/node_modules + key: node-modules-backend-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci + + - name: Restore data-provider build cache + id: cache-data-provider + uses: actions/cache@v4 + with: + path: packages/data-provider/dist + key: build-data-provider-${{ runner.os }}-${{ hashFiles('packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }} + + - name: Build data-provider + if: steps.cache-data-provider.outputs.cache-hit != 'true' + run: npm run build:data-provider + + - name: Restore data-schemas build cache + id: cache-data-schemas + uses: actions/cache@v4 + with: + path: packages/data-schemas/dist + key: build-data-schemas-${{ runner.os }}-${{ hashFiles('packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }} + + - name: Build data-schemas + if: steps.cache-data-schemas.outputs.cache-hit != 'true' + run: npm run build:data-schemas + + - name: Restore api build cache + id: cache-api + uses: actions/cache@v4 + with: + path: packages/api/dist + key: build-api-${{ runner.os }}-${{ hashFiles('packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/tsdown.config.mjs', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json') }} + + - name: Build api + if: steps.cache-api.outputs.cache-hit != 'true' + run: npm run build:api + + - name: Run agents integration tests (in-process MongoDB) + working-directory: packages/api + env: + NODE_ENV: test + run: npm run test:agents-integration diff --git a/package-lock.json b/package-lock.json index 280c7906ae..99384d9869 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20806,7 +20806,8 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/babel-plugin-replace-ts-export-assignment/-/babel-plugin-replace-ts-export-assignment-0.0.2.tgz", "integrity": "sha512-BiTEG2Ro+O1spuheL5nB289y37FFmz0ISE6GjpNCG2JuA/WNcuEHSYw01+vN8quGf208sID3FnZFDwVyqX18YQ==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/babel-plugin-root-import": { "version": "6.6.0", @@ -42920,6 +42921,7 @@ "@types/winston": "^2.4.4", "@types/yauzl": "^2.10.3", "aws-sdk-client-mock": "^4.1.0", + "babel-plugin-replace-ts-export-assignment": "^0.0.2", "dedent": "^1.5.3", "get-stream": "^6.0.1", "jest": "^30.2.0", diff --git a/packages/api/package.json b/packages/api/package.json index bb4f40f02f..bb184ba750 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -33,6 +33,7 @@ "test:cache-integration:stream": "jest --testPathPatterns=\"src/stream/.*\\.stream_integration\\.spec\\.ts$\" --coverage=false --runInBand --forceExit", "test:cache-integration": "npm run test:cache-integration:core && npm run test:cache-integration:cluster && npm run test:cache-integration:mcp && npm run test:cache-integration:stream", "test:s3-integration": "jest --testPathPatterns=\"src/storage/s3/.*\\.integration\\.spec\\.ts$\" --coverage=false --runInBand", + "test:agents-integration": "jest --testPathPatterns=\"src/agents/.*\\.integration\\.spec\\.ts$\" --coverage=false --runInBand --forceExit", "verify": "npm run test:ci", "b:clean": "bun run rimraf dist", "b:build": "bun run b:clean && bun run tsdown", @@ -77,6 +78,7 @@ "@types/winston": "^2.4.4", "@types/yauzl": "^2.10.3", "aws-sdk-client-mock": "^4.1.0", + "babel-plugin-replace-ts-export-assignment": "^0.0.2", "dedent": "^1.5.3", "get-stream": "^6.0.1", "jest": "^30.2.0", diff --git a/packages/api/src/agents/checkpointer.integration.spec.ts b/packages/api/src/agents/checkpointer.integration.spec.ts index cc67b96817..f617a745bc 100644 --- a/packages/api/src/agents/checkpointer.integration.spec.ts +++ b/packages/api/src/agents/checkpointer.integration.spec.ts @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import { logger } from '@librechat/data-schemas'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb'; import { emptyCheckpoint, ERROR, INTERRUPT } from '@langchain/langgraph-checkpoint'; @@ -6,6 +7,8 @@ import { getAgentCheckpointer, deleteAgentCheckpoint, deleteAgentCheckpoints, + LazyMongoSaver, + CheckpointTooLargeError, __resetCheckpointerForTests, } from './checkpointer'; @@ -446,3 +449,197 @@ describe('LazyMongoSaver (lazy persistence — mongodb-memory-server)', () => { expect(effects).toEqual({ a: 1, c: 1 }); // siblings did NOT re-execute }); }); + +describe('LazyMongoSaver checkpoint size guard (mongodb-memory-server integration)', () => { + // Guards the single-document ceiling: a checkpoint embeds the whole message history, so a + // large conversation can serialize past MongoDB's 16 MB limit. The guard measures the + // serialized state on the persist path, WARNS past a soft threshold, and REJECTS past a hard + // limit BEFORE the write — a typed CheckpointTooLargeError instead of a raw BSONObjectTooLarge. + // Thresholds are shrunk here so payloads stay tiny while exercising the real serde + Mongo. + const clientForSaver = () => + // mongoose vends the live MongoClient; the driver type resolves to a different `mongodb` copy + // than checkpoint-mongodb's, so the cast mirrors buildMongoSaver in checkpointer.ts. + mongoose.connection.getClient() as unknown as ConstructorParameters< + typeof MongoDBSaver + >[0]['client']; + + const makeSaver = (overrides?: { warnBytes?: number; hardLimitBytes?: number }) => + new LazyMongoSaver({ + client: clientForSaver(), + checkpointCollectionName: 'agent_checkpoints', + checkpointWritesCollectionName: 'agent_checkpoint_writes', + ttl: 3600, + ...overrides, + }); + + /** Seed an interrupt anchor for a checkpoint whose serialized state is inflated to ~`payloadBytes`. */ + async function seedSizedInterrupt(saver: LazyMongoSaver, threadId: string, payloadBytes: number) { + const checkpoint = emptyCheckpoint(); + checkpoint.channel_values = { messages: 'x'.repeat(payloadBytes) }; + await saver.putWrites( + { configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } }, + [[INTERRUPT, 'approve?']], + 'task-1', + ); + const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } }; + const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} }; + return { checkpoint, config, metadata }; + } + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('persists a checkpoint under the soft threshold', async () => { + const saver = makeSaver({ warnBytes: 5_000, hardLimitBytes: 50_000 }); + await saver.setup(); + const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`; + const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 100); + + await saver.put(config, checkpoint, metadata); + + expect(await saver.getTuple(readConfig(threadId))).toBeDefined(); + }); + + it('persists but WARNS when a checkpoint crosses the soft threshold', async () => { + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => logger); + const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 50_000 }); + await saver.setup(); + const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`; + const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 2_000); + + await saver.put(config, checkpoint, metadata); + + expect(await saver.getTuple(readConfig(threadId))).toBeDefined(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('soft threshold')); + }); + + it('REFUSES to persist and throws CheckpointTooLargeError over the hard limit', async () => { + jest.spyOn(logger, 'error').mockImplementation(() => logger); + const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 2_000 }); + await saver.setup(); + const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`; + const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 10_000); + + await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf( + CheckpointTooLargeError, + ); + + // Nothing durable was written for that thread — getTuple reads the checkpoint document. + expect(await saver.getTuple(readConfig(threadId))).toBeUndefined(); + expect( + await mongoose.connection.db!.collection('agent_checkpoints').countDocuments({ + thread_id: threadId, + }), + ).toBe(0); + }); + + it('counts metadata toward the ceiling — refuses when the checkpoint is under but metadata pushes over', async () => { + // MongoDBSaver stores the serialized checkpoint AND metadata in one document, so a + // just-under-limit checkpoint with large metadata still overflows. The guard must catch it + // as a typed CheckpointTooLargeError, not let it fall through to a raw BSONObjectTooLarge. + jest.spyOn(logger, 'error').mockImplementation(() => logger); + const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 2_000 }); + await saver.setup(); + const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`; + + const checkpoint = emptyCheckpoint(); + checkpoint.channel_values = { messages: 'x'.repeat(400) }; // checkpoint alone well UNDER 2 KB + await saver.putWrites( + { configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } }, + [[INTERRUPT, 'approve?']], + 'task-1', + ); + const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } }; + // Metadata alone pushes checkpoint + metadata over the 2 KB hard limit. + const metadata = { + source: 'input' as const, + step: -1, + writes: { payload: 'y'.repeat(4_000) }, + parents: {}, + }; + + await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf( + CheckpointTooLargeError, + ); + expect(await saver.getTuple(readConfig(threadId))).toBeUndefined(); + }); + + it('counts metadata_search (the raw metadata copy) — refuses when serialized-only would pass', async () => { + // MongoDBSaver stores `metadata_search: metadata` — the WHOLE raw metadata a + // SECOND time in the same document. Sized so checkpoint + serialized metadata is + // UNDER the limit but adding the raw metadata_search copy pushes it over: the guard + // must count that copy, else the doc slips the preflight and overflows on the wire. + jest.spyOn(logger, 'error').mockImplementation(() => logger); + const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 6_000 }); + await saver.setup(); + const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`; + + const checkpoint = emptyCheckpoint(); + checkpoint.channel_values = { messages: 'x'.repeat(200) }; + await saver.putWrites( + { configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } }, + [[INTERRUPT, 'approve?']], + 'task-1', + ); + const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } }; + // ~3 KB metadata: checkpoint + serialized metadata (~3.2 KB) is under 6 KB, but the + // raw metadata_search copy (~another 3 KB) pushes checkpoint+metadata+metadata_search + // over 6 KB. The pre-fix guard (checkpoint + serialized metadata only) would pass here. + const metadata = { + source: 'loop' as const, + step: 1, + writes: { payload: 'y'.repeat(3_000) }, + parents: {}, + }; + + await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf( + CheckpointTooLargeError, + ); + expect(await saver.getTuple(readConfig(threadId))).toBeUndefined(); + }); + + it('flushes bookkeeping parked during the size-serialization window (not dropped on resume)', async () => { + // `put` consumes the write anchor, then AWAITS `assertCheckpointFitsDocument` (serialization). + // A bookkeeping-only putWrites dispatched in that window sees no anchor and no persisted + // marker, so it parks — and must be flushed once the checkpoint persists, or a resume + // re-executes the completed sibling. + const NO_WRITES = '__no_writes__'; + const saver = makeSaver({ warnBytes: 5_000, hardLimitBytes: 50_000 }); + await saver.setup(); + const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`; + const checkpoint = emptyCheckpoint(); + const writeCfg = { + configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id }, + }; + await saver.putWrites(writeCfg, [[INTERRUPT, 'approve?']], 'task-gate'); + + // Pause the checkpoint serialization inside `put` so the bookkeeping batch lands mid-window. + const serde = (saver as unknown as { serde: { dumpsTyped: (v: unknown) => unknown } }).serde; + const realDumps = serde.dumpsTyped.bind(serde); + let releaseGate!: () => void; + const gate = new Promise((resolve) => { + releaseGate = resolve; + }); + let paused = false; + jest.spyOn(serde, 'dumpsTyped').mockImplementation(async (value: unknown) => { + if (!paused) { + paused = true; + await gate; // hold inside assertCheckpointFitsDocument (anchor consumed, not yet persisted) + } + return realDumps(value); + }); + + const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } }; + const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} }; + const putPromise = saver.put(config, checkpoint, metadata); // suspends at the gate + await saver.putWrites(writeCfg, [[NO_WRITES, null]], 'task-sibling'); // parks in the window + releaseGate(); + await putPromise; + + const tuple = await saver.getTuple(readConfig(threadId)); + const channels = (tuple?.pendingWrites ?? []).map((w) => w[1]); + expect(channels).toContain(INTERRUPT); + expect(channels).toContain(NO_WRITES); // flushed, not dropped + }); +}); diff --git a/packages/api/src/agents/checkpointer.ts b/packages/api/src/agents/checkpointer.ts index 79b6954346..52a644d2cb 100644 --- a/packages/api/src/agents/checkpointer.ts +++ b/packages/api/src/agents/checkpointer.ts @@ -124,6 +124,69 @@ interface BufferedWriteBatch { batches: Array<{ config: RunnableConfig; writes: PendingWrite[]; taskId: string }>; } +/** + * MongoDB's hard per-document ceiling. A checkpoint whose serialized state pushes its + * document past this cannot be stored — the driver throws `BSONObjectTooLarge` (code 10334). + */ +const MAX_BSON_DOCUMENT_BYTES = 16 * 1024 * 1024; + +/** + * Headroom reserved below {@link MAX_BSON_DOCUMENT_BYTES} for a checkpoint document's + * non-state fields (ids, metadata, `metadata_search`, BSON framing). The serialized + * `checkpoint` blob dominates the document; this margin covers everything else so the guard + * rejects before Mongo does — with a legible error instead of a raw driver failure. + */ +const CHECKPOINT_SIZE_HEADROOM_BYTES = 1024 * 1024; + +/** + * Reject a checkpoint whose serialized state exceeds this. The pause is unrecoverable either + * way (the document can't be written), so failing here as a typed {@link CheckpointTooLargeError} + * turns an opaque `BSONObjectTooLarge` crash into an actionable one. + */ +export const CHECKPOINT_HARD_LIMIT_BYTES: number = + MAX_BSON_DOCUMENT_BYTES - CHECKPOINT_SIZE_HEADROOM_BYTES; + +/** + * Warn once a persisted checkpoint crosses this soft threshold (~50% of the ceiling), so a + * conversation's checkpoint growth is visible in logs well before it reaches the hard limit. + */ +export const CHECKPOINT_WARN_BYTES: number = 8 * 1024 * 1024; + +/** + * A HITL checkpoint whose serialized state exceeds {@link CHECKPOINT_HARD_LIMIT_BYTES} — more + * than MongoDB can hold in a single document. Thrown BEFORE the doomed write so the run fails + * with a clear, typed message instead of a raw driver `BSONObjectTooLarge`. The pause cannot be + * persisted regardless of how it is handled upstream; a durable resume is impossible for this turn. + */ +export class CheckpointTooLargeError extends Error { + readonly code = 'CHECKPOINT_TOO_LARGE'; + constructor( + readonly bytes: number, + readonly limit: number, + readonly threadId?: string, + ) { + const mb = (n: number): string => (n / 1024 / 1024).toFixed(1); + super( + `Checkpoint state is ${mb(bytes)} MB, over the ${mb(limit)} MB limit for a durable pause. ` + + 'This conversation carries too much state to pause for input — large tool outputs or ' + + 'inlined media are the usual cause. Start a new conversation or reduce context.', + ); + this.name = 'CheckpointTooLargeError'; + } +} + +/** + * Construction options for {@link LazyMongoSaver}: the base saver options plus optional + * size-guard overrides. The overrides default to the module thresholds and exist so tests can + * exercise the guard at small sizes; production always uses the defaults. + */ +export type LazyMongoSaverOptions = ConstructorParameters[0] & { + /** Soft warn threshold in bytes. Defaults to {@link CHECKPOINT_WARN_BYTES}. */ + warnBytes?: number; + /** Hard reject limit in bytes. Defaults to {@link CHECKPOINT_HARD_LIMIT_BYTES}. */ + hardLimitBytes?: number; +}; + export class LazyMongoSaver extends MongoDBSaver { /** checkpoint id → time the resumable `putWrites` anchoring it arrived; consumed by `put`. */ private readonly writeAnchorIds = new Map(); @@ -133,6 +196,18 @@ export class LazyMongoSaver extends MongoDBSaver { /** checkpoint id → bookkeeping batches parked until the checkpoint persists or is discarded. */ private readonly bufferedBookkeeping = new Map(); + /** Soft threshold (bytes) past which a persisted checkpoint is warned about. */ + private readonly warnBytes: number; + /** Hard limit (bytes) past which a checkpoint is refused with {@link CheckpointTooLargeError}. */ + private readonly hardLimitBytes: number; + + constructor(options: LazyMongoSaverOptions) { + const { warnBytes, hardLimitBytes, ...mongoOptions } = options; + super(mongoOptions); + this.warnBytes = warnBytes ?? CHECKPOINT_WARN_BYTES; + this.hardLimitBytes = hardLimitBytes ?? CHECKPOINT_HARD_LIMIT_BYTES; + } + override async putWrites( config: RunnableConfig, writes: PendingWrite[], @@ -174,15 +249,9 @@ export class LazyMongoSaver extends MongoDBSaver { // wrongly discarded. this.recordWriteAnchor(checkpointId); try { - const buffered = this.bufferedBookkeeping.get(checkpointId); - this.bufferedBookkeeping.delete(checkpointId); - if (buffered) { - // The checkpoint's fate is now "persist" — flush the bookkeeping batches that - // arrived before this anchor so the stored pending writes are complete. - await Promise.all( - buffered.batches.map((b) => super.putWrites(b.config, b.writes, b.taskId)), - ); - } + // The checkpoint's fate is now "persist" — flush the bookkeeping batches that + // arrived before this anchor so the stored pending writes are complete. + await this.flushBufferedBookkeeping(checkpointId); return await super.putWrites(config, writes, taskId); } catch (err) { // The write batch never landed — best-effort un-anchor so the concurrent `put` doesn't @@ -203,9 +272,17 @@ export class LazyMongoSaver extends MongoDBSaver { // Carries a resumable write (interrupt / real-channel delta anchor) — persist so resume // can read it, and remember the id briefly so any bookkeeping batch dispatched after // this `put` is forwarded rather than parked. + await this.assertCheckpointFitsDocument(config, checkpoint, metadata); sweepStale(this.persistedIds, (t) => t); this.persistedIds.set(checkpoint.id, Date.now()); - return super.put(config, checkpoint, metadata); + const persisted = await super.put(config, checkpoint, metadata); + // `assertCheckpointFitsDocument` awaits a (potentially slow) serialization AFTER the + // anchor was consumed above but BEFORE `persistedIds` was set — a bookkeeping-only + // `putWrites` dispatched in that window sees neither marker and parks its batch. Flush + // it now that the checkpoint is persisted; without this the marker is dropped and a + // resume can re-execute already-completed work. + await this.flushBufferedBookkeeping(checkpoint.id); + return persisted; } // No resumable writes ⇒ a clean exit (a non-paused completion, a resumed turn's clean // finish, or an error-only turn): discard, and drop the parked bookkeeping batches with @@ -233,6 +310,74 @@ export class LazyMongoSaver extends MongoDBSaver { sweepStale(this.writeAnchorIds, (t) => t); this.writeAnchorIds.set(checkpointId, Date.now()); } + + /** + * Forward the bookkeeping batches parked for `checkpointId` while its fate was undecided, + * now that the checkpoint is being persisted. Snapshot-and-delete before awaiting so a batch + * that arrives afterwards can't be double-forwarded — by then the anchor/persisted marker is + * set, so it forwards directly instead of parking. Shared by the anchoring `putWrites` and by + * `put` (for a batch parked during the size-check serialization window). + */ + private async flushBufferedBookkeeping(checkpointId: string): Promise { + const buffered = this.bufferedBookkeeping.get(checkpointId); + if (!buffered) { + return; + } + this.bufferedBookkeeping.delete(checkpointId); + await Promise.all(buffered.batches.map((b) => super.putWrites(b.config, b.writes, b.taskId))); + } + + /** + * Measure the checkpoint's serialized size on the persist path and act on it: `debug`-log it, + * `warn` past {@link warnBytes}, and throw {@link CheckpointTooLargeError} past + * {@link hardLimitBytes} — BEFORE the write, so an oversize pause fails legibly rather than as a + * raw `BSONObjectTooLarge`. Serializes with the same `serde` the base `put` uses, so the measured + * bytes match what would be stored. The extra serialization runs only on the (rare) HITL pause + * path — never the clean-exit common path, which is discarded before reaching here. + */ + private async assertCheckpointFitsDocument( + config: RunnableConfig, + checkpoint: Checkpoint, + metadata: CheckpointMetadata, + ): Promise { + // `MongoDBSaver.put` writes THREE size-bearing fields into the same + // `agent_checkpoints` document: the serialized `checkpoint`, the serialized + // `metadata`, AND `metadata_search` — the WHOLE raw `metadata` object stored a + // second time as a queryable BSON subdocument (`metadata_search: metadata`). + // So a large `metadata` (e.g. `metadata.writes` holding a big tool result) is + // counted twice on the wire. Measuring only checkpoint + serialized metadata + // let such a document pass the preflight while `metadata_search` pushed the + // actual BSON past 16 MB — the raw `BSONObjectTooLarge` this guard exists to + // prevent. Add the raw metadata's BSON size; the headroom now only has to + // cover ids and BSON framing. + const [, serializedCheckpoint] = await this.serde.dumpsTyped(checkpoint); + const [, serializedMetadata] = await this.serde.dumpsTyped(metadata); + const metadataSearchBytes = mongoose.mongo.BSON.calculateObjectSize( + metadata as unknown as Record, + ); + const bytes = + serializedCheckpoint.byteLength + serializedMetadata.byteLength + metadataSearchBytes; + const threadId = config.configurable?.thread_id as string | undefined; + const mb = (n: number): string => (n / 1024 / 1024).toFixed(1); + if (bytes > this.hardLimitBytes) { + // The anchoring write row was already persisted by `putWrites`; the pre-run prune and Mongo + // TTL reclaim it. Drop any parked bookkeeping so it doesn't linger in memory. + this.bufferedBookkeeping.delete(checkpoint.id); + logger.error( + `[checkpointer] HITL checkpoint for thread ${threadId ?? 'unknown'} is ${mb(bytes)} MB, over the ${mb(this.hardLimitBytes)} MB durable-pause limit; refusing the write (a document past 16 MB cannot be stored in MongoDB).`, + ); + throw new CheckpointTooLargeError(bytes, this.hardLimitBytes, threadId); + } + if (bytes >= this.warnBytes) { + logger.warn( + `[checkpointer] HITL checkpoint for thread ${threadId ?? 'unknown'} is ${mb(bytes)} MB, past the ${mb(this.warnBytes)} MB soft threshold (hard limit ${mb(this.hardLimitBytes)} MB) — approaching MongoDB's single-document ceiling.`, + ); + return; + } + logger.debug( + `[checkpointer] Persisting HITL checkpoint for thread ${threadId ?? 'unknown'}: ${bytes} bytes`, + ); + } } /**