From 4da2b9d2a3ec20960f0200f177fbac35240eccf6 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:49:10 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=9A=20refactor:=20Separate=20Checkpoin?= =?UTF-8?q?t=20Storage=20From=20Graph=20Runtime=20(#18029)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: Separate Checkpoint Storage From Graph Runtime * fix: Preserve Checkpoint Index Retry Guard * test: initialize checkpoint identity within test lifecycle * test: Register Persistence Boot Spec With Source Guards --------- Co-authored-by: Lia --- .github/guard-specs.json | 1 + .../agents/checkpointer.integration.spec.ts | 4 +- packages/api/src/agents/checkpointer.ts | 429 +++------------- .../checkpoints/actor.integration.spec.ts | 17 +- .../api/src/agents/checkpoints/deletion.ts | 257 +++------- .../api/src/agents/checkpoints/pruning.ts | 97 +--- .../api/src/agents/checkpoints/saver.spec.ts | 102 ++++ packages/api/src/agents/checkpoints/saver.ts | 406 ++++++--------- .../api/src/agents/checkpoints/storage.ts | 119 +---- packages/api/src/stream/checkpoints.ts | 25 +- .../data-schemas/src/checkpoints/catalog.ts | 129 +++++ .../data-schemas/src/checkpoints/database.ts | 16 + .../data-schemas/src/checkpoints/identity.ts | 20 + .../data-schemas/src/checkpoints/indexes.ts | 45 ++ .../data-schemas/src/checkpoints/journal.ts | 164 ++++++ .../data-schemas/src/checkpoints/limits.ts | 50 ++ .../src/checkpoints/mongo.spec.ts | 263 ++++++++++ .../data-schemas/src/checkpoints/mongo.ts | 484 ++++++++++++++++++ .../data-schemas/src/checkpoints/pruning.ts | 111 ++++ .../data-schemas/src/checkpoints/types.ts | 159 ++++++ packages/data-schemas/src/index.ts | 20 + 21 files changed, 1879 insertions(+), 1039 deletions(-) create mode 100644 packages/api/src/agents/checkpoints/saver.spec.ts create mode 100644 packages/data-schemas/src/checkpoints/catalog.ts create mode 100644 packages/data-schemas/src/checkpoints/database.ts create mode 100644 packages/data-schemas/src/checkpoints/identity.ts create mode 100644 packages/data-schemas/src/checkpoints/indexes.ts create mode 100644 packages/data-schemas/src/checkpoints/journal.ts create mode 100644 packages/data-schemas/src/checkpoints/limits.ts create mode 100644 packages/data-schemas/src/checkpoints/mongo.spec.ts create mode 100644 packages/data-schemas/src/checkpoints/mongo.ts create mode 100644 packages/data-schemas/src/checkpoints/pruning.ts create mode 100644 packages/data-schemas/src/checkpoints/types.ts diff --git a/.github/guard-specs.json b/.github/guard-specs.json index 7505565a95..443506c8e6 100644 --- a/.github/guard-specs.json +++ b/.github/guard-specs.json @@ -7,6 +7,7 @@ ], "api/server/experimental.spec.js": ["api/server/experimental.js", "api/server/index.js"], "api/server/index.spec.js": ["api/server/index.js", "api/server/experimental.js"], + "api/server/persistence.spec.js": ["api/server/index.js", "api/server/experimental.js"], "api/server/routes/agents/__tests__/openapi.contract.spec.js": [ "packages/api/openapi/agents.openapi.json" ], diff --git a/packages/api/src/agents/checkpointer.integration.spec.ts b/packages/api/src/agents/checkpointer.integration.spec.ts index c19fcc433e..826b556dbd 100644 --- a/packages/api/src/agents/checkpointer.integration.spec.ts +++ b/packages/api/src/agents/checkpointer.integration.spec.ts @@ -50,7 +50,7 @@ const readConfig = (threadId: string, checkpointNamespace = '') => ({ * persists only checkpoints seeded this way (a bare `put` is a clean exit → discarded). */ async function seedInterruptCheckpoint( - saver: MongoDBSaver, + saver: Pick, threadId: string, checkpointNamespace = '', interruptId = 'interrupt-current', @@ -830,7 +830,7 @@ describe('LazyMongoSaver (lazy persistence — mongodb-memory-server)', () => { const { config, checkpoint, metadata } = putArgs(threadId); const spy = jest - .spyOn(MongoDBSaver.prototype, 'putWrites') + .spyOn(saver!.repository, 'saveWrites') .mockRejectedValueOnce(new Error('transient mongo failure')); try { await expect( diff --git a/packages/api/src/agents/checkpointer.ts b/packages/api/src/agents/checkpointer.ts index 3aa3be9223..75ce243620 100644 --- a/packages/api/src/agents/checkpointer.ts +++ b/packages/api/src/agents/checkpointer.ts @@ -1,7 +1,13 @@ -import mongoose from 'mongoose'; import { INTERRUPT } from '@langchain/langgraph-checkpoint'; -import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb'; -import { buildIndexWithRetry, isIndexBuildInProgress, logger } from '@librechat/data-schemas'; +import { + logger, + setupCheckpointIndexes, + CheckpointTooLargeError, + CHECKPOINT_HARD_LIMIT_BYTES, + CHECKPOINT_WARN_BYTES, + createMongoCheckpointRepository, + isCheckpointDatabaseAvailable, +} from '@librechat/data-schemas'; import type { Checkpoint, CheckpointListOptions, @@ -9,17 +15,22 @@ import type { CheckpointTuple, PendingWrite, } from '@langchain/langgraph-checkpoint'; +import type { + CheckpointRepository, + CheckpointGeneration, + CheckpointReference, +} from '@librechat/data-schemas'; +import type { MongoDBSaverParams } from '@langchain/langgraph-checkpoint-mongodb'; import type { BaseMessage } from '@librechat/agents/langchain/messages'; import type { TCheckpointerConfig } from 'librechat-data-provider'; -import type { IndexBuildOptions } from '@librechat/data-schemas'; import type { RunnableConfig } from '@langchain/core/runnables'; +import type { OwnedCheckpointSaverOptions } from './checkpoints/saver'; import type { ResolvedCheckpointerConfig } from './checkpoints/config'; import { OwnedMongoSaver, LIBRECHAT_CHECKPOINT_OWNER_KEY, LIBRECHAT_LEGACY_CHECKPOINT_KEY, } from './checkpoints/saver'; -import { checkpointOwnerNamespacePrefix } from '../stream/checkpoints'; import { resolveCheckpointerConfig } from './checkpoints/config'; export { @@ -36,6 +47,13 @@ export { export type { ResolvedCheckpointerConfig } from './checkpoints/config'; export { DEFAULT_CHECKPOINT_TTL_SECONDS } from '../stream/checkpoints'; +export { + setupCheckpointIndexes, + CheckpointTooLargeError, + CHECKPOINT_HARD_LIMIT_BYTES, + CHECKPOINT_WARN_BYTES, +} from '@librechat/data-schemas'; + /** * LangGraph reserves `checkpoint_ns` for nested graph namespaces and forcibly * resets a non-empty value to `''` for every root invocation. Carry LibreChat's @@ -136,20 +154,6 @@ function fromStorageCheckpointTuple( }; } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -/** Mongo filter matching a generation's root and every nested graph namespace. */ -function generationNamespaceFilter(checkpointNamespace: string): string | { $regex: string } { - if (checkpointNamespace === '') { - return ''; - } - return { - $regex: `^${escapeRegExp(checkpointNamespace)}(?:\\${CHECKPOINT_NAMESPACE_SEPARATOR}|$)`, - }; -} - /** * Durable checkpointing for human-in-the-loop (HITL) resume. * @@ -260,7 +264,7 @@ function hasResumableWrite(writes: PendingWrite[]): boolean { * checkpoint (and the now-discarded error-only checkpoint) was only ever written-then-pruned, so * not writing it is observationally equivalent; the pre-run prune + Mongo TTL remain the * backstops. The saver overrides every config-bearing read/write path to apply - * generation storage scoping; `deleteThread` and `setup` remain inherited. + * generation storage scoping; `deleteThread` and `setup` delegate to the injected repository. */ /** A bookkeeping-only pending-write batch held until its checkpoint's fate is decided. */ interface BufferedWriteBatch { @@ -268,70 +272,21 @@ interface BufferedWriteBatch { batches: Array<{ config: RunnableConfig; writes: PendingWrite[]; taskId: string }>; } -/** - * MongoDB's hard per-document ceiling. A checkpoint whose serialized state pushes its - * document past this cannot be stored — the driver throws `BSONObjectTooLarge` (code 10334). - */ -const MAX_BSON_DOCUMENT_BYTES = 16 * 1024 * 1024; - -/** - * Headroom reserved below {@link MAX_BSON_DOCUMENT_BYTES} for a checkpoint document's - * non-state fields (ids, metadata, `metadata_search`, BSON framing). The serialized - * `checkpoint` blob dominates the document; this margin covers everything else so the guard - * rejects before Mongo does — with a legible error instead of a raw driver failure. - */ -const CHECKPOINT_SIZE_HEADROOM_BYTES = 1024 * 1024; - -/** - * Reject a checkpoint whose serialized state exceeds this. The pause is unrecoverable either - * way (the document can't be written), so failing here as a typed {@link CheckpointTooLargeError} - * turns an opaque `BSONObjectTooLarge` crash into an actionable one. - */ -export const CHECKPOINT_HARD_LIMIT_BYTES: number = - MAX_BSON_DOCUMENT_BYTES - CHECKPOINT_SIZE_HEADROOM_BYTES; - -/** - * Warn once a persisted checkpoint crosses this soft threshold (~50% of the ceiling), so a - * conversation's checkpoint growth is visible in logs well before it reaches the hard limit. - */ -export const CHECKPOINT_WARN_BYTES: number = 8 * 1024 * 1024; - -/** - * A durable checkpoint whose serialized state exceeds {@link CHECKPOINT_HARD_LIMIT_BYTES} — more - * than MongoDB can hold in a single document. Thrown BEFORE the doomed write so the run fails - * with a clear, typed message instead of a raw driver `BSONObjectTooLarge`. The checkpoint cannot - * be persisted regardless of how it is handled upstream, so a durable resume is impossible. - */ -export class CheckpointTooLargeError extends Error { - readonly code = 'CHECKPOINT_TOO_LARGE'; - constructor( - readonly bytes: number, - readonly limit: number, - readonly threadId?: string, - ) { - const mb = (n: number): string => (n / 1024 / 1024).toFixed(1); - super( - `Checkpoint state is ${mb(bytes)} MB, over the ${mb(limit)} MB durable limit. ` + - 'This conversation carries too much state to resume — large tool outputs or inlined ' + - 'media are the usual cause. Start a new conversation or reduce context.', - ); - this.name = 'CheckpointTooLargeError'; - } -} - /** * Construction options for {@link LazyMongoSaver}: the base saver options plus optional * size-guard overrides. The overrides default to the module thresholds and exist so tests can * exercise the guard at small sizes; production always uses the defaults. */ -export type LazyMongoSaverOptions = ConstructorParameters[0] & { +interface LazyCheckpointLimits { /** Trusted root conversation for child payload cleanup, never derived from child IDs. */ conversationId?: string; /** Soft warn threshold in bytes. Defaults to {@link CHECKPOINT_WARN_BYTES}. */ warnBytes?: number; /** Hard reject limit in bytes. Defaults to {@link CHECKPOINT_HARD_LIMIT_BYTES}. */ hardLimitBytes?: number; -}; +} + +export type LazyMongoSaverOptions = MongoDBSaverParams & LazyCheckpointLimits; export class LazyMongoSaver extends OwnedMongoSaver { /** checkpoint id → time the resumable `putWrites` anchoring it arrived; consumed by `put`. */ @@ -347,7 +302,9 @@ export class LazyMongoSaver extends OwnedMongoSaver { /** Hard limit (bytes) past which a checkpoint is refused with {@link CheckpointTooLargeError}. */ private readonly hardLimitBytes: number; - constructor(options: LazyMongoSaverOptions) { + constructor(options: OwnedCheckpointSaverOptions & LazyCheckpointLimits); + constructor(options: LazyMongoSaverOptions); + constructor(options: OwnedCheckpointSaverOptions & LazyCheckpointLimits) { const { warnBytes, hardLimitBytes, ...mongoOptions } = options; super(mongoOptions); this.warnBytes = warnBytes ?? CHECKPOINT_WARN_BYTES; @@ -538,9 +495,7 @@ export class LazyMongoSaver extends OwnedMongoSaver { // cover ids and BSON framing. const [, serializedCheckpoint] = await this.serde.dumpsTyped(checkpoint); const [, serializedMetadata] = await this.serde.dumpsTyped(metadata); - const metadataSearchBytes = mongoose.mongo.BSON.calculateObjectSize( - metadata as unknown as Record, - ); + const metadataSearchBytes = this.repository.measureMetadata(metadata); const bytes = serializedCheckpoint.byteLength + serializedMetadata.byteLength + metadataSearchBytes; const threadId = config.configurable?.thread_id as string | undefined; @@ -590,14 +545,7 @@ function sweepStale(map: Map, timeOf: (value: T) => number): void * Legacy jobs share storage, so cleanup deletes only this captured set; a later * replacement's fresh checkpoint ids cannot be removed by the delayed cleanup. */ -export interface AgentCheckpointGeneration { - threadId: string; - /** Nonempty saver-level generation scope. Missing means a legacy - * thread-wide capture; an empty string is invalid because it would omit the - * legacy generation's nested LangGraph namespaces during deletion. */ - checkpointNamespace?: string; - checkpointIds: string[]; -} +export type AgentCheckpointGeneration = CheckpointGeneration; /** Approval-window milliseconds from the resolved config; drives pending-action expiry. */ export function getApprovalTtlMs(cfg: TCheckpointerConfig | undefined): number { @@ -664,7 +612,7 @@ export async function hasDurableAgentInterruptCheckpoint( * One saver per process, built lazily on first use so `setup()` (index creation) * runs exactly once. Keyed by the resolved settings so a config change rebuilds. */ -let saverPromise: Promise | undefined; +let saverPromise: Promise | undefined; let cachedKey: string | undefined; function settingsKey(resolved: ResolvedCheckpointerConfig): string { @@ -682,12 +630,12 @@ function settingsKey(resolved: ResolvedCheckpointerConfig): string { export async function getAgentCheckpointer( cfg: TCheckpointerConfig | undefined, conversationId?: string, -): Promise { +): Promise { const resolved = resolveCheckpointerConfig(cfg); if (resolved.type === 'memory') { return undefined; } - if (mongoose.connection.readyState !== 1) { + if (!isCheckpointDatabaseAvailable()) { logger.warn( '[checkpointer] Mongoose not connected; durable agent continuations will use an in-process checkpointer this turn and will not survive a restart or resolve on another replica.', ); @@ -703,11 +651,7 @@ export async function getAgentCheckpointer( return saver && conversationId ? createMongoSaver(resolved, conversationId) : saver; } -export interface AgentEventCheckpointReference { - threadId: string; - checkpointId: string; - checkpointNs: string; -} +export type AgentEventCheckpointReference = CheckpointReference; export interface AgentEventCheckpointMessageOverlay { source: string; @@ -815,20 +759,7 @@ export async function deleteAgentEventCheckpointReference( if (resolved.type === 'memory') { return false; } - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) { - throw new Error('Mongo checkpoint storage is unavailable'); - } - const exact = { - thread_id: reference.threadId, - checkpoint_ns: reference.checkpointNs, - checkpoint_id: reference.checkpointId, - ...(owner == null - ? { lc_owner: { $exists: false } } - : { $or: [{ lc_owner: owner }, { lc_owner: { $exists: false } }] }), - }; - await db.collection(resolved.checkpointWritesCollectionName).deleteMany(exact); - await db.collection(resolved.checkpointCollectionName).deleteOne(exact); + await checkpointRepository(resolved).deleteReference(reference, owner); return true; } @@ -840,25 +771,7 @@ export async function deleteAgentEventCheckpointReferences( ): Promise { const resolved = resolveCheckpointerConfig(cfg); if (resolved.type === 'memory' || references.length === 0) return; - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) - throw new Error('Checkpoint database is unavailable'); - for (let offset = 0; offset < references.length; offset += 256) { - const exact = { - $and: [ - { $or: [{ lc_owner: owner }, { lc_owner: { $exists: false } }] }, - { - $or: references.slice(offset, offset + 256).map((reference) => ({ - thread_id: reference.threadId, - checkpoint_ns: reference.checkpointNs, - checkpoint_id: reference.checkpointId, - })), - }, - ], - }; - await db.collection(resolved.checkpointWritesCollectionName).deleteMany(exact); - await db.collection(resolved.checkpointCollectionName).deleteMany(exact); - } + await checkpointRepository(resolved).deleteReferences(references, owner); } /** Reads the terminal checkpoint produced inside one invocation namespace. */ @@ -888,78 +801,24 @@ export async function captureAgentEventCheckpoint( : null; } -/** Longest wait for a peer's checkpoint index build before the saver proceeds without it. */ -const CHECKPOINT_INDEX_BUILD_DEADLINE_MS = 120_000; - -/** - * `MongoDBSaver.setup()` starts the compound and TTL builds of each collection - * concurrently and reports failures instead of throwing. Amazon DocumentDB - * admits one index build per collection at a time, so there the second build of - * each pair is rejected (code 40333) and the saver would otherwise run with its - * TTL index missing — checkpoints then never expire. `setup()` is idempotent (an - * index that already exists is a no-op), so re-running it lets one more build - * through per pass until every index exists. Every other error is returned for - * the caller to log, exactly as `setup()` reports it — including those reported - * beside a conflict that outlasts the deadline. A rejection of `setup()` itself - * propagates, so the caller keeps its in-process fallback. - */ -export async function setupCheckpointIndexes( - saver: Pick, - options: IndexBuildOptions = {}, -): Promise { - let companions: Error[] = []; - try { - return await buildIndexWithRetry( - async () => { - const errors = await saver.setup(); - const blocked = errors.find(isIndexBuildInProgress); - if (blocked == null) { - return errors; - } - companions = errors.filter((error) => !isIndexBuildInProgress(error)); - throw blocked; - }, - 'MongoDBSaver.setup()', - { peerBuildDeadlineMs: CHECKPOINT_INDEX_BUILD_DEADLINE_MS, ...options }, - ); - } catch (error) { - if (!isIndexBuildInProgress(error)) { - throw error; - } - return [...companions, error instanceof Error ? error : new Error(String(error))]; - } +function checkpointRepository(resolved: ResolvedCheckpointerConfig): CheckpointRepository { + return createMongoCheckpointRepository({ + checkpointCollectionName: resolved.checkpointCollectionName, + checkpointWritesCollectionName: resolved.checkpointWritesCollectionName, + ttl: resolved.ttlSeconds, + }); } function createMongoSaver( resolved: ResolvedCheckpointerConfig, conversationId?: string, ): LazyMongoSaver { - return new LazyMongoSaver({ - conversationId, - // mongoose vends the live MongoClient; reuse it instead of opening a second - // connection. The driver type is structurally identical but resolves to a - // different `mongodb` copy than checkpoint-mongodb's, hence the cast. - client: mongoose.connection.getClient() as unknown as ConstructorParameters< - typeof MongoDBSaver - >[0]['client'], - // MongoDBSaver calls MongoClient.db(dbName). Passing no name makes that - // resolve from the driver's URI default, which is not guaranteed to be the - // database Mongoose selected (for example when Mongoose connected with a - // dbName override). Every capture/delete path below uses connection.db, so - // bind the saver to that exact database as well or a pause can be written to - // one database while LibreChat looks for it in another. - dbName: mongoose.connection.db?.databaseName, - checkpointCollectionName: resolved.checkpointCollectionName, - checkpointWritesCollectionName: resolved.checkpointWritesCollectionName, - // TTL index on `upserted_at`: an unresolved paused run is reclaimed after the - // approval window, so a forgotten approval can never leak checkpoints forever. - ttl: resolved.ttlSeconds, - }); + return new LazyMongoSaver({ conversationId, repository: checkpointRepository(resolved) }); } async function buildMongoSaver( resolved: ResolvedCheckpointerConfig, -): Promise { +): Promise { try { const saver = createMongoSaver(resolved); const errors = await setupCheckpointIndexes(saver); @@ -969,49 +828,7 @@ async function buildMongoSaver( errors, ); } - await Promise.all( - [resolved.checkpointCollectionName, resolved.checkpointWritesCollectionName].map( - async (name) => { - try { - await buildIndexWithRetry( - () => mongoose.connection.db!.collection(name).createIndex({ checkpoint_ns: 1 }), - `${name}.checkpoint_ns`, - ); - } catch (error) { - logger.warn('[checkpointer] Owner cleanup index unavailable:', error); - } - }, - ), - ); - await Promise.all( - [resolved.checkpointCollectionName, resolved.checkpointWritesCollectionName].map( - async (name) => { - try { - await buildIndexWithRetry( - () => - mongoose.connection.db!.collection(name).createIndex({ lc_owner: 1, thread_id: 1 }), - `${name}.lc_owner`, - ); - } catch (error) { - logger.warn('[checkpointer] Payload ownership index unavailable:', error); - } - }, - ), - ); - await Promise.all( - [resolved.checkpointCollectionName, resolved.checkpointWritesCollectionName].map( - async (name) => { - try { - await buildIndexWithRetry( - () => mongoose.connection.db!.collection(name).createIndex({ lc_parent_thread: 1 }), - `${name}.lc_parent_thread`, - ); - } catch (error) { - logger.warn('[checkpointer] Child cleanup index unavailable:', error); - } - }, - ), - ); + await saver.repository.setupOwnershipIndexes(); logger.info('[checkpointer] Durable Mongo checkpointer ready for agent continuation'); return saver; } catch (err) { @@ -1026,10 +843,6 @@ async function buildMongoSaver( } } -function legacyCheckpointStorageFilter() { - return { checkpoint_ns: { $not: /^lcg:v2:/ }, lc_owner: { $exists: false } }; -} - /** * Snapshot the durable checkpoint ids that belong to the generation about to * resume. Capture this before atomically claiming the paused job; a replacement @@ -1060,29 +873,11 @@ export async function captureAgentCheckpointGeneration( } try { const saver = await getAgentCheckpointer(cfg); - const db = mongoose.connection.db; - if (!saver || !db) { - return generation; - } - const resolved = resolveCheckpointerConfig(cfg); - const checkpoints = await db - .collection<{ checkpoint_id?: string }>(resolved.checkpointCollectionName) - .find( - { - thread_id: threadId, - ...(namespaceScoped - ? { checkpoint_ns: generationNamespaceFilter(requestedNamespace) } - : legacyCheckpointStorageFilter()), - }, - { projection: { _id: 0, checkpoint_id: 1 } }, - ) - .toArray(); - generation.checkpointIds = checkpoints.reduce((ids, checkpoint) => { - if (typeof checkpoint.checkpoint_id === 'string') { - ids.push(checkpoint.checkpoint_id); - } - return ids; - }, []); + if (!saver) return generation; + generation.checkpointIds = await saver.repository.captureGeneration( + threadId, + namespaceScoped ? requestedNamespace : undefined, + ); } catch (err) { logger.warn( `[checkpointer] Failed to capture checkpoint generation for thread ${threadId}:`, @@ -1122,77 +917,10 @@ export async function deleteAgentCheckpoint( return; } try { - if (generation) { - if (generation.threadId !== threadId || generation.checkpointIds.length === 0) { - return; - } - if ( - Object.prototype.hasOwnProperty.call(generation, 'checkpointNamespace') && - (generation.checkpointNamespace ?? '') === '' - ) { - throw new Error( - 'Legacy checkpoint cleanup requires a thread-wide captured generation without an empty namespace marker', - ); - } - const db = mongoose.connection.db; - if (!db) { - if (options?.throwOnError) throw new Error('Checkpoint database is unavailable'); - return; - } - const resolved = resolveCheckpointerConfig(cfg); - const filter = { - thread_id: threadId, - ...(Object.prototype.hasOwnProperty.call(generation, 'checkpointNamespace') - ? { checkpoint_ns: generationNamespaceFilter(generation.checkpointNamespace ?? '') } - : legacyCheckpointStorageFilter()), - checkpoint_id: { $in: generation.checkpointIds }, - }; - await Promise.all([ - db.collection(resolved.checkpointCollectionName).deleteMany(filter), - db.collection(resolved.checkpointWritesCollectionName).deleteMany(filter), - ]); - return; - } - if (Object.prototype.hasOwnProperty.call(options ?? {}, 'checkpointNamespace')) { - const checkpointNamespace = options?.checkpointNamespace ?? ''; - /** An explicit empty namespace denotes a legacy/pre-isolation job, not - * an immutable storage scope. Thread-wide deletion could erase a newer - * v2 replacement, while filtering `checkpoint_ns: ''` would strand the - * legacy job's nested subgraphs. Such callers must capture a thread-wide - * immutable id set, verify their job epoch after that capture, and pass - * the resulting `generation` above. */ - if (checkpointNamespace === '') { - throw new Error( - 'Legacy checkpoint cleanup requires a captured checkpoint generation, not an empty namespace', - ); - } - const db = mongoose.connection.db; - if (!db) { - if (options?.throwOnError) throw new Error('Checkpoint database is unavailable'); - return; - } - const resolved = resolveCheckpointerConfig(cfg); - const filter = { - thread_id: threadId, - checkpoint_ns: generationNamespaceFilter(checkpointNamespace), - }; - await Promise.all([ - db.collection(resolved.checkpointCollectionName).deleteMany(filter), - db.collection(resolved.checkpointWritesCollectionName).deleteMany(filter), - ]); - return; - } - const db = mongoose.connection.db; - if (!db) { - if (options?.throwOnError) throw new Error('Checkpoint database is unavailable'); - return; - } - const resolved = resolveCheckpointerConfig(cfg); - const filter = { thread_id: threadId, ...legacyCheckpointStorageFilter() }; - await Promise.all([ - db.collection(resolved.checkpointCollectionName).deleteMany(filter), - db.collection(resolved.checkpointWritesCollectionName).deleteMany(filter), - ]); + const namespace = Object.prototype.hasOwnProperty.call(options ?? {}, 'checkpointNamespace') + ? (options?.checkpointNamespace ?? '') + : undefined; + await saver.repository.deleteGeneration(threadId, generation, namespace); } catch (err) { logger.warn(`[checkpointer] Failed to delete checkpoints for thread ${threadId}:`, err); if (options?.throwOnError) { @@ -1210,18 +938,7 @@ export async function deleteOwnedActorCheckpointScope( ): Promise { const resolved = resolveCheckpointerConfig(cfg); if (resolved.type === 'memory') return; - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) - throw new Error('Checkpoint database is unavailable'); - const filter = { - thread_id: `${owner}${threadId}`, - checkpoint_ns: generationNamespaceFilter(`${owner}${checkpointNs}`), - lc_owner: owner, - }; - await Promise.all([ - db.collection(resolved.checkpointCollectionName).deleteMany(filter), - db.collection(resolved.checkpointWritesCollectionName).deleteMany(filter), - ]); + await checkpointRepository(resolved).deleteOwnedScope(threadId, checkpointNs, owner); } /** Delete durable rows by authenticated ownership, independently of job/receipt lifetimes. */ @@ -1238,35 +955,7 @@ export async function deleteOwnedAgentCheckpoints( if (resolved.type === 'memory' || conversationIds?.length === 0) { return; } - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) { - throw new Error('Checkpoint database is unavailable'); - } - const owner = checkpointOwnerNamespacePrefix(userId, tenantId); - const ownership = { $or: [{ checkpoint_ns: { $regex: `^${owner}` } }, { lc_owner: owner }] }; - const ids = conversationIds == null ? undefined : [...new Set(conversationIds)]; - const batchSize = 128; - for (let offset = 0; offset < (ids?.length ?? 1); offset += batchSize) { - const filter = { - ...ownership, - ...(ids && { - $and: [ - { - $or: [ - { - thread_id: { - $in: ids.slice(offset, offset + batchSize).flatMap((id) => [id, `${owner}${id}`]), - }, - }, - { lc_parent_thread: { $in: ids.slice(offset, offset + batchSize) } }, - ], - }, - ], - }), - }; - await db.collection(resolved.checkpointCollectionName).deleteMany(filter); - await db.collection(resolved.checkpointWritesCollectionName).deleteMany(filter); - } + await checkpointRepository(resolved).deleteOwned(userId, tenantId, conversationIds); } /** Test-only: drop the memoized saver so a fresh build is forced. */ diff --git a/packages/api/src/agents/checkpoints/actor.integration.spec.ts b/packages/api/src/agents/checkpoints/actor.integration.spec.ts index 30e24d7d0a..13060d03e3 100644 --- a/packages/api/src/agents/checkpoints/actor.integration.spec.ts +++ b/packages/api/src/agents/checkpoints/actor.integration.spec.ts @@ -89,7 +89,7 @@ test('late legacy writes and another tagged owner survive owned scope deletion', test('old raw readers cannot see isolated owner payloads', async () => { const id = await write('event-actor/head', 'owner'); - const tuple = await MongoDBSaver.prototype.getTuple.call((await getAgentCheckpointer(cfg))!, { + const tuple = await rawSaver().getTuple({ configurable: { thread_id: 'actor-thread', checkpoint_ns: 'event-actor/head', @@ -99,7 +99,7 @@ test('old raw readers cannot see isolated owner payloads', async () => { expect(tuple).toBeUndefined(); const saver = (await getAgentCheckpointer(cfg))!; expect( - await MongoDBSaver.prototype.getTuple.call(saver, { + await rawSaver().getTuple({ configurable: { thread_id: 'actor-thread', checkpoint_ns: 'event-actor/head' }, }), ).toBeUndefined(); @@ -119,7 +119,7 @@ test('raw thread cleanup cannot remove isolated actor checkpoints or pending wri const id = await write('event-actor/head', 'owner'); await write('event-actor/head'); const saver = (await getAgentCheckpointer(cfg))!; - await MongoDBSaver.prototype.deleteThread.call(saver, 'actor-thread'); + await rawSaver().deleteThread('actor-thread'); const tuple = await saver.getTuple(config('event-actor/head', 'owner')); expect(tuple?.checkpoint.id).toBe(id); expect(tuple?.pendingWrites).toHaveLength(1); @@ -394,3 +394,14 @@ test('custom checkpoint collections carry owner authority without separate maint expect(await mongoose.connection.db!.collection('tenant_cp').countDocuments()).toBe(0); expect(await mongoose.connection.db!.collection('tenant_writes').countDocuments()).toBe(0); }); + +function rawSaver(): MongoDBSaver { + return new MongoDBSaver({ + client: mongoose.connection.getClient() as unknown as ConstructorParameters< + typeof MongoDBSaver + >[0]['client'], + dbName: mongoose.connection.db!.databaseName, + checkpointCollectionName: 'agent_checkpoints', + checkpointWritesCollectionName: 'agent_checkpoint_writes', + }); +} diff --git a/packages/api/src/agents/checkpoints/deletion.ts b/packages/api/src/agents/checkpoints/deletion.ts index 36fcd1a0cc..1855269364 100644 --- a/packages/api/src/agents/checkpoints/deletion.ts +++ b/packages/api/src/agents/checkpoints/deletion.ts @@ -1,31 +1,35 @@ -import mongoose from 'mongoose'; -import { createHash, randomUUID } from 'crypto'; -import { logger } from '@librechat/data-schemas'; -import type { TCheckpointerConfig } from 'librechat-data-provider'; -import type { AgentEventCheckpointReference } from '../checkpointer'; -import type { ResolvedCheckpointerConfig } from './config'; -import type { CheckpointStorageRecord } from './storage'; +import { randomUUID } from 'crypto'; import { - checkpointStorageKey, - checkpointStorageConfigs, - ownedCheckpointReferences, - CHECKPOINT_STORAGE_COLLECTION, -} from './storage'; + logger, + createCheckpointCatalog, + createCheckpointDeletionJournal, + checkpointDeletionTarget, + createActorCheckpointPruning, +} from '@librechat/data-schemas'; +import type { + CheckpointCatalog, + CheckpointDeletionJournal, + ActorCheckpointPruning, + CheckpointDeletionTarget, +} from '@librechat/data-schemas'; +import type { TCheckpointerConfig } from 'librechat-data-provider'; import { deleteOwnedAgentCheckpoints, deleteAgentEventCheckpointReferences } from '../checkpointer'; +import { checkpointStorageKey, checkpointStorageConfigs } from './storage'; import { checkpointOwnerNamespacePrefix } from '../../stream/checkpoints'; import { historicalActorReferences } from './pruning'; -interface DeletionTarget { - _id: string; - version: string; - threadId: string; - checkpoint?: AgentEventCheckpointReference; - userId: string; - tenantId?: string; - storage: ResolvedCheckpointerConfig; +interface CheckpointDeletionStorage { + catalog: CheckpointCatalog; + journal: CheckpointDeletionJournal; + pruning: ActorCheckpointPruning; +} +function deletionStorage(): CheckpointDeletionStorage { + return { + catalog: createCheckpointCatalog(), + journal: createCheckpointDeletionJournal(), + pruning: createActorCheckpointPruning(), + }; } - -const DELETION_COLLECTION = 'agent_checkpoint_deletions'; export interface CheckpointDeletion { conversationIds(): string[]; @@ -34,9 +38,6 @@ export interface CheckpointDeletion { acknowledge(): Promise; } -const hash = (value: string | null) => - createHash('sha256').update(JSON.stringify(value)).digest('hex'); - /** Durable deletion intent preserves cascade identity after topology is removed. * These records contain no checkpoint payload and exist only during deletion, * not once per generation. They must survive until both drain and cleanup succeed. */ @@ -45,82 +46,52 @@ export async function openCheckpointDeletion( tenantId: string | undefined, rootConversationId: string | undefined, cfg?: TCheckpointerConfig, + persistence: CheckpointDeletionStorage = deletionStorage(), ): Promise { if (!userId) { throw new Error('Checkpoint deletion requires an owner'); } - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) { - throw new Error('Checkpoint database is unavailable'); - } - const collection = db.collection(DELETION_COLLECTION); + const { catalog, journal, pruning } = persistence; const storageKey = checkpointStorageKey; const ownerPrefix = checkpointOwnerNamespacePrefix(userId, tenantId); const tenants = tenantId ? [tenantId, undefined] : [undefined]; - const prefixes = tenants.map((tenant) => { - const prefix = checkpointOwnerNamespacePrefix(userId, tenant); - return rootConversationId == null - ? prefix - : `${prefix}[0-9a-f]{64}:${hash(rootConversationId)}:`; - }); - const retained = await collection - .find({ $or: prefixes.map((prefix) => ({ _id: { $regex: `^${prefix}` } })) }) - .toArray(); - const targets = new Map(retained.map((target) => [target._id, target])); + const retained = await journal.retained(userId, tenantId, rootConversationId); + const targets = new Map(retained.map((target) => [target.id, target])); const stores = new Map(retained.map((target) => [storageKey(target.storage), target.storage])); async function refreshStores() { - for (const storage of await checkpointStorageConfigs(userId, tenantId, cfg)) { + for (const storage of await checkpointStorageConfigs(userId, tenantId, cfg, catalog)) { stores.set(storageKey(storage), storage); } } await refreshStores(); const version = randomUUID(); + const scope = { userId, tenantId, rootConversationId }; + const targetFor = ( + storage: CheckpointDeletionTarget['storage'], + threadId: string, + checkpoint?: CheckpointDeletionTarget['checkpoint'], + proof?: 'owned' | 'historical', + ) => checkpointDeletionTarget(scope, storage, threadId, version, checkpoint, proof); const batchSize = 256; - async function persist(batch: DeletionTarget[]) { - await collection.bulkWrite( - batch.map((target) => ({ - updateOne: { - filter: { _id: target._id }, - update: { - $set: { - version, - threadId: target.threadId, - userId, - tenantId, - storage: target.storage, - ...(target.checkpoint && { checkpoint: target.checkpoint }), - }, - }, - upsert: true, - }, - })), - ); - for (const target of batch) targets.set(target._id, target); + async function persist(batch: CheckpointDeletionTarget[]) { + await journal.remember(batch); + for (const target of batch) targets.set(target.id, target); } const conversationIds = () => [ ...new Set([...targets.values()].map((target) => target.threadId)), ]; async function captureOwned(ids: readonly string[] | undefined) { - let batch: DeletionTarget[] = []; - for (const [key, storage] of stores) { - const rootPrefix = `${ownerPrefix}${key}:${hash(rootConversationId ?? null)}:`; - for await (const { checkpoint, conversationId } of ownedCheckpointReferences( + let batch: CheckpointDeletionTarget[] = []; + for (const storage of stores.values()) { + for await (const { checkpoint, conversationId } of catalog.references( userId, tenantId, ids, storage, )) { - batch.push({ - _id: `${rootPrefix}${hash(conversationId)}:${hash(JSON.stringify([checkpoint.threadId, checkpoint.checkpointNs, checkpoint.checkpointId]))}`, - version, - threadId: conversationId, - userId, - tenantId, - storage, - checkpoint, - }); + batch.push(targetFor(storage, conversationId, checkpoint, 'owned')); if (batch.length === batchSize) { await persist(batch); batch = []; @@ -135,33 +106,19 @@ export async function openCheckpointDeletion( await refreshStores(); for (let offset = 0; offset < ids.length; offset += batchSize) { const threads = ids.slice(offset, offset + batchSize); - for (const [key, storage] of stores) { - const rootPrefix = `${ownerPrefix}${key}:${hash(rootConversationId ?? null)}:`; - await persist( - threads.map((threadId) => ({ - _id: `${rootPrefix}${hash(threadId)}`, - version, - threadId, - userId, - tenantId, - storage, - })), - ); + for (const storage of stores.values()) { + await persist(threads.map((threadId) => targetFor(storage, threadId))); } await captureOwned(threads); - let batch: DeletionTarget[] = []; - for await (const checkpoint of historicalActorReferences(userId, tenantId, threads)) { - for (const [key, storage] of stores) { - const rootPrefix = `${ownerPrefix}${key}:${hash(rootConversationId ?? null)}:`; - batch.push({ - _id: `${rootPrefix}${hash(checkpoint.threadId)}:${hash(JSON.stringify([checkpoint.checkpointNs, checkpoint.checkpointId]))}`, - version, - threadId: checkpoint.threadId, - userId, - tenantId, - storage, - checkpoint, - }); + let batch: CheckpointDeletionTarget[] = []; + for await (const checkpoint of historicalActorReferences( + userId, + tenantId, + threads, + pruning, + )) { + for (const storage of stores.values()) { + batch.push(targetFor(storage, checkpoint.threadId, checkpoint, 'historical')); if (batch.length === batchSize) { await persist(batch); batch = []; @@ -174,7 +131,9 @@ export async function openCheckpointDeletion( async cleanup() { await refreshStores(); await captureOwned(rootConversationId == null ? undefined : conversationIds()); - const groups = new Map([...stores.keys()].map((key) => [key, [] as DeletionTarget[]])); + const groups = new Map( + [...stores.keys()].map((key) => [key, [] as CheckpointDeletionTarget[]]), + ); for (const target of targets.values()) groups.get(storageKey(target.storage))!.push(target); for (const [key, storage] of stores) { const group = groups.get(key)!; @@ -196,27 +155,19 @@ export async function openCheckpointDeletion( async acknowledge() { const receipts = [...targets.values()]; for (let offset = 0; offset < receipts.length; offset += batchSize) { - await collection.deleteMany({ - $or: receipts.slice(offset, offset + batchSize).map(({ _id, version: captured }) => ({ - _id, - version: captured, - })), - }); + await journal.acknowledge(receipts.slice(offset, offset + batchSize)); } if (rootConversationId != null) return; - const pending = await collection.findOne( - { $or: prefixes.map((prefix) => ({ _id: { $regex: `^${prefix}` } })) }, - { projection: { _id: 1 } }, - ); - if (pending != null) throw new Error('Checkpoint deletion intent is still pending'); + if (await journal.hasPending(userId, tenantId, rootConversationId)) + throw new Error('Checkpoint deletion intent is still pending'); // Owner-wide callers retain their deletion fence through acknowledgement. - const descriptors = [...stores.keys()].flatMap((key) => - tenants.map((tenant) => `${checkpointOwnerNamespacePrefix(userId, tenant)}${key}`), - ); + const descriptors = [...stores.values()]; for (let offset = 0; offset < descriptors.length; offset += batchSize) { - await db.collection(CHECKPOINT_STORAGE_COLLECTION).deleteMany({ - _id: { $in: descriptors.slice(offset, offset + batchSize) }, - }); + await catalog.forgetDescriptors( + userId, + tenantId, + descriptors.slice(offset, offset + batchSize), + ); } }, }; @@ -225,69 +176,16 @@ export async function openCheckpointDeletion( /** Replay captured identities after topology deletion; never sweep new generations. */ export function createCheckpointDeletionReclaimer( getOwnerJobs: (userId: string, tenantId?: string) => Promise, + journal: CheckpointDeletionJournal = createCheckpointDeletionJournal(), ): (limit: number, activity?: { found: boolean }) => Promise { let after: string | undefined; return async (limit, activity) => { if (!Number.isSafeInteger(limit) || limit <= 0) throw new Error('Invalid reclamation limit'); - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) - throw new Error('Checkpoint database is unavailable'); - const collection = db.collection(DELETION_COLLECTION); - const targets = await collection - .find(after == null ? {} : { _id: { $gt: after } }) - .sort({ _id: 1 }) - .limit(limit) - .toArray(); + const targets = await journal.scan(limit, after); if (activity != null && targets.length > 0) activity.found = true; - after = targets.length === limit ? targets[targets.length - 1]._id : undefined; + after = targets.length === limit ? targets[targets.length - 1].id : undefined; const jobsByOwner = new Map>(); - const hasPersistence = async (target: DeletionTarget): Promise => { - const { userId, tenantId, threadId, checkpoint, storage } = target; - const owners = (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => - checkpointOwnerNamespacePrefix(userId, tenant), - ); - const user = mongoose.isValidObjectId(userId) ? new mongoose.Types.ObjectId(userId) : userId; - const siblingCollections = [ - [mongoose.models.Conversation?.collection.name ?? 'conversations', userId], - [mongoose.models.Message?.collection.name ?? 'messages', userId], - [mongoose.models.ToolCall?.collection.name ?? 'toolcalls', user], - [mongoose.models.SharedLink?.collection.name ?? 'sharedlinks', userId], - ] as const; - const siblings = siblingCollections.map(([name, owner]) => - db - .collection(name) - .findOne({ user: owner, conversationId: threadId }, { projection: { _id: 1 } }), - ); - const payload = - storage.type === 'memory' - ? [] - : [storage.checkpointCollectionName, storage.checkpointWritesCollectionName].map((name) => - db.collection(name).findOne( - { - $or: [ - ...owners.map((owner) => ({ - thread_id: { $in: [threadId, `${owner}${threadId}`] }, - $or: [{ checkpoint_ns: { $regex: `^${owner}` } }, { lc_owner: owner }], - })), - ...(checkpoint == null - ? [] - : [ - { - thread_id: checkpoint.threadId, - checkpoint_ns: checkpoint.checkpointNs, - checkpoint_id: checkpoint.checkpointId, - $or: [{ lc_owner: { $in: owners } }, { lc_owner: { $exists: false } }], - }, - ]), - ], - }, - { projection: { _id: 1 } }, - ), - ); - return (await Promise.all([...siblings, ...payload])).some((row) => row != null); - }; - const results = await Promise.allSettled( targets.map(async (target) => { const owner = checkpointOwnerNamespacePrefix(target.userId, target.tenantId); @@ -298,13 +196,7 @@ export function createCheckpointDeletionReclaimer( } if ((await jobs).length > 0) return 0; if (target.checkpoint != null) { - const conversation = await db - .collection(mongoose.models.Conversation?.collection.name ?? 'conversations') - .findOne( - { user: target.userId, conversationId: target.threadId }, - { projection: { _id: 1 } }, - ); - if (conversation == null) { + if (!(await journal.hasConversation(target.userId, target.threadId))) { for (const tenant of target.tenantId ? [target.tenantId, undefined] : [undefined]) { await deleteAgentEventCheckpointReferences( [target.checkpoint], @@ -314,9 +206,8 @@ export function createCheckpointDeletionReclaimer( } } } - if (await hasPersistence(target)) return 0; - return (await collection.deleteOne({ _id: target._id, version: target.version })) - .deletedCount; + if (await journal.hasPersistence(target)) return 0; + return journal.reclaim(target); }), ); const failures = results.filter((result) => result.status === 'rejected'); diff --git a/packages/api/src/agents/checkpoints/pruning.ts b/packages/api/src/agents/checkpoints/pruning.ts index 18a1fddf9e..c0c9c28a26 100644 --- a/packages/api/src/agents/checkpoints/pruning.ts +++ b/packages/api/src/agents/checkpoints/pruning.ts @@ -1,27 +1,15 @@ -import mongoose from 'mongoose'; -import type { IConversation } from '@librechat/data-schemas'; +import { createActorCheckpointPruning } from '@librechat/data-schemas'; +import type { ActorCheckpointPruning } from '@librechat/data-schemas'; import type { AgentEventCheckpointReference } from '../checkpointer'; -function conversations() { - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) { - throw new Error('Actor checkpoint cleanup requires Mongo storage'); - } - return db.collection & { tenantId?: string | null }>( - mongoose.models.Conversation?.collection.name ?? 'conversations', - ); -} - export async function acknowledgeActorPruning( user: string, tenantId: string | undefined, conversationId: string, reference: AgentEventCheckpointReference, + storage: ActorCheckpointPruning = createActorCheckpointPruning(), ): Promise { - await conversations().updateOne( - { user, tenantId: tenantId ?? null, conversationId }, - { $pull: { agentEventActorCleanup: reference } }, - ); + await storage.acknowledge(user, tenantId, conversationId, reference); } export async function drainActorPruning( @@ -29,82 +17,19 @@ export async function drainActorPruning( tenantId: string | undefined, conversationId: string, remove: (reference: AgentEventCheckpointReference) => Promise, + storage: ActorCheckpointPruning = createActorCheckpointPruning(), ): Promise { - const conversation = await conversations().findOne( - { user, tenantId: tenantId ?? null, conversationId }, - { projection: { agentEventActorCleanup: 1 } }, - ); - for (const reference of conversation?.agentEventActorCleanup ?? []) { + for (const reference of await storage.pending(user, tenantId, conversationId)) { await remove(reference); - await acknowledgeActorPruning(user, tenantId, conversationId, reference); + await storage.acknowledge(user, tenantId, conversationId, reference); } } -/** Read hidden historical evidence while the owner-filtered conversation still exists. */ -export async function* historicalActorReferences( +export function historicalActorReferences( user: string, tenantId: string | undefined, conversationIds: readonly string[] | undefined, -): AsyncGenerator { - const ids = conversationIds == null ? undefined : [...new Set(conversationIds)]; - for (let offset = 0; offset < (ids?.length ?? 1); offset += 256) { - const cursor = conversations().find( - { - user, - tenantId: tenantId ? { $in: [tenantId, null] } : null, - subagentThread: { $exists: true }, - ...(ids && { conversationId: { $in: ids.slice(offset, offset + 256) } }), - }, - { - projection: { - conversationId: 1, - 'agentEventActor.checkpoint': 1, - 'agentEventActor.previousCheckpoint': 1, - 'agentEventActorSuspension.suspension.checkpoint': 1, - 'agentEventActorSuspension.suspension.invocation.fork': 1, - 'agentEventActorSuspension.suspension.invocation.base.checkpoint': 1, - 'agentEventActorReconciliations.checkpoint': 1, - agentEventActorCleanup: 1, - }, - }, - ); - for await (const conversation of cursor) { - const suspension = conversation.agentEventActorSuspension?.suspension; - const candidates = [ - conversation.agentEventActor?.checkpoint, - conversation.agentEventActor?.previousCheckpoint, - suspension?.checkpoint, - suspension?.invocation?.fork, - suspension?.invocation?.base?.checkpoint, - ...(conversation.agentEventActorReconciliations ?? []).map((entry) => entry.checkpoint), - ...(conversation.agentEventActorCleanup ?? []), - ]; - const seen = new Set(); - for (const reference of candidates) { - if ( - typeof reference?.checkpointId !== 'string' || - reference.checkpointId.length === 0 || - typeof reference.checkpointNs !== 'string' || - !reference.checkpointNs.startsWith('event-actor/') || - reference.threadId !== conversation.conversationId - ) { - continue; - } - const key = JSON.stringify([ - reference.threadId, - reference.checkpointNs, - reference.checkpointId, - ]); - if (seen.has(key)) { - continue; - } - seen.add(key); - yield { - threadId: reference.threadId, - checkpointNs: reference.checkpointNs, - checkpointId: reference.checkpointId, - }; - } - } - } + storage: ActorCheckpointPruning = createActorCheckpointPruning(), +): AsyncIterable { + return storage.historical(user, tenantId, conversationIds); } diff --git a/packages/api/src/agents/checkpoints/saver.spec.ts b/packages/api/src/agents/checkpoints/saver.spec.ts new file mode 100644 index 0000000000..dbb88ed4aa --- /dev/null +++ b/packages/api/src/agents/checkpoints/saver.spec.ts @@ -0,0 +1,102 @@ +import { emptyCheckpoint, INTERRUPT } from '@langchain/langgraph-checkpoint'; +import type { CheckpointRepository } from '@librechat/data-schemas'; +import { + LazyMongoSaver, + LIBRECHAT_CHECKPOINT_NAMESPACE_KEY, + LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY, + LIBRECHAT_CHECKPOINT_OWNER_KEY, +} from '../checkpointer'; + +function storage(): CheckpointRepository { + return { + setup: jest.fn(async () => []), + setupOwnershipIndexes: jest.fn(async () => {}), + get: jest.fn(async () => null), + list: jest.fn(async function* () {}), + readWrites: jest.fn(async () => []), + save: jest.fn(async () => {}), + saveWrites: jest.fn(async () => {}), + measureMetadata: jest.fn(() => 0), + deleteThread: jest.fn(async () => {}), + captureGeneration: jest.fn(async () => []), + deleteGeneration: jest.fn(async () => {}), + deleteReference: jest.fn(async () => {}), + deleteReferences: jest.fn(async () => {}), + deleteOwnedScope: jest.fn(async () => {}), + deleteOwned: jest.fn(async () => {}), + }; +} +const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} }; + +test('ordinary clean chat never reaches the repository or serializes metadata', async () => { + const repository = storage(); + const saver = new LazyMongoSaver({ repository, conversationId: 'root' }); + await saver.put({ configurable: { thread_id: 'root' } }, emptyCheckpoint(), metadata); + expect(repository.save).not.toHaveBeenCalled(); + expect(repository.saveWrites).not.toHaveBeenCalled(); + expect(repository.measureMetadata).not.toHaveBeenCalled(); +}); + +test.each([true, false])( + 'child persistence is injected byte storage with writes-first=%s', + async (writesFirst) => { + const repository = storage(); + const saver = new LazyMongoSaver({ repository, conversationId: 'trusted-root' }); + const checkpoint = emptyCheckpoint(); + const config = { + configurable: { + thread_id: 'subagent:not-the-root', + checkpoint_ns: '', + checkpoint_id: checkpoint.id, + [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'generation', + [LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY]: 'owner', + [LIBRECHAT_CHECKPOINT_OWNER_KEY]: 'owner', + }, + }; + const put = () => saver.put(config, checkpoint, metadata); + const writes = () => saver.putWrites(config, [[INTERRUPT, { id: 'pause' }]], 'task'); + if (writesFirst) { + await writes(); + await put(); + } else { + await put(); + await writes(); + } + expect(repository.save).toHaveBeenCalledWith( + expect.objectContaining({ + threadId: 'subagent:not-the-root', + checkpointNs: 'generation', + owner: 'owner', + }), + expect.objectContaining({ + checkpoint: { type: expect.any(String), bytes: expect.any(Uint8Array) }, + metadata: { type: expect.any(String), bytes: expect.any(Uint8Array) }, + }), + { rootConversationId: 'trusted-root', storageOwner: 'owner' }, + ); + expect(repository.saveWrites).toHaveBeenCalledWith( + expect.objectContaining({ checkpointId: checkpoint.id, checkpointNs: 'generation' }), + [ + expect.objectContaining({ + taskId: 'task', + channel: INTERRUPT, + value: { type: expect.any(String), bytes: expect.any(Uint8Array) }, + }), + ], + { replace: true, rootConversationId: 'trusted-root', storageOwner: 'owner' }, + ); + }, +); + +test('failed interrupt persistence unanchors an ordinary checkpoint through the injected boundary', async () => { + const repository = storage(); + const saver = new LazyMongoSaver({ repository }); + const checkpoint = emptyCheckpoint(); + const config = { configurable: { thread_id: 'root', checkpoint_id: checkpoint.id } }; + jest.mocked(repository.saveWrites).mockRejectedValueOnce(new Error('unavailable')); + await expect(saver.putWrites(config, [[INTERRUPT, 'pause']], 'task')).rejects.toThrow( + 'unavailable', + ); + await saver.put(config, checkpoint, metadata); + expect(repository.save).not.toHaveBeenCalled(); +}); diff --git a/packages/api/src/agents/checkpoints/saver.ts b/packages/api/src/agents/checkpoints/saver.ts index 2fe275f65e..316915494f 100644 --- a/packages/api/src/agents/checkpoints/saver.ts +++ b/packages/api/src/agents/checkpoints/saver.ts @@ -1,5 +1,5 @@ -import { WRITES_IDX_MAP } from '@langchain/langgraph-checkpoint'; -import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb'; +import { createMongoCheckpointRepository } from '@librechat/data-schemas'; +import { BaseCheckpointSaver, WRITES_IDX_MAP } from '@langchain/langgraph-checkpoint'; import type { Checkpoint, CheckpointMetadata, @@ -7,140 +7,106 @@ import type { CheckpointListOptions, CheckpointPendingWrite, PendingWrite, + SerializerProtocol, } from '@langchain/langgraph-checkpoint'; +import type { + CheckpointRepository, + CheckpointIdentity, + SerializedCheckpoint, + MongoCheckpointRepositoryOptions, +} from '@librechat/data-schemas'; +import type { MongoDBSaverParams } from '@langchain/langgraph-checkpoint-mongodb'; import type { RunnableConfig } from '@langchain/core/runnables'; -import type { Binary } from 'mongodb'; -import type { Filter } from 'mongodb'; - -import type { CheckpointStorageRecord } from './storage'; -import { - CHECKPOINT_STORAGE_COLLECTION, - LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY, - checkpointStorageKey, -} from './storage'; +import { LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY } from './storage'; export const LIBRECHAT_CHECKPOINT_OWNER_KEY = '__librechat_checkpoint_owner'; export const LIBRECHAT_LEGACY_CHECKPOINT_KEY = '__librechat_legacy_checkpoint_id'; -interface CheckpointRow { - thread_id: string; - checkpoint_ns: string; - checkpoint_id: string; - parent_checkpoint_id?: string; - lc_owner?: string; - type: string; - checkpoint: Binary; - metadata: Binary; - metadata_search: CheckpointMetadata; -} - -interface WriteRow { - thread_id: string; - checkpoint_ns: string; - checkpoint_id: string; - task_id: string; - idx: number; - channel: string; - type: string; - value: Binary; - lc_owner?: string; -} +export type OwnedCheckpointSaverOptions = ( + | MongoDBSaverParams + | { repository: CheckpointRepository; client?: never } +) & { + conversationId?: string; + serde?: SerializerProtocol; +}; function ownerOf(config: RunnableConfig): string | undefined { const owner = config.configurable?.[LIBRECHAT_CHECKPOINT_OWNER_KEY]; return typeof owner === 'string' && owner.length > 0 ? owner : undefined; } - -function identity(config: RunnableConfig): { - thread_id: string; - checkpoint_ns: string; - checkpoint_id?: string; -} { - const thread_id = config.configurable?.thread_id; - const checkpoint_ns = config.configurable?.checkpoint_ns ?? ''; - const checkpoint_id = config.configurable?.checkpoint_id; +function identity(config: RunnableConfig): CheckpointIdentity { + const threadId = config.configurable?.thread_id; + const checkpointNs = config.configurable?.checkpoint_ns ?? ''; + const checkpointId = config.configurable?.checkpoint_id; if ( - typeof thread_id !== 'string' || - typeof checkpoint_ns !== 'string' || - (checkpoint_id != null && typeof checkpoint_id !== 'string') - ) { + typeof threadId !== 'string' || + typeof checkpointNs !== 'string' || + (checkpointId != null && typeof checkpointId !== 'string') + ) throw new Error('Invalid owned checkpoint identity'); - } - return { thread_id, checkpoint_ns, ...(checkpoint_id == null ? {} : { checkpoint_id }) }; + return { + threadId, + checkpointNs, + checkpointId, + owner: ownerOf(config), + legacyCheckpointId: config.configurable?.[LIBRECHAT_LEGACY_CHECKPOINT_KEY], + }; } -/** Isolate physical storage by owner while preserving logical SDK references. */ -export class OwnedMongoSaver extends MongoDBSaver { +/** LangGraph serialization and logical identity mapping, independent of the storage engine. */ +export class OwnedMongoSaver extends BaseCheckpointSaver { + readonly repository: CheckpointRepository; private readonly conversationId?: string; - - constructor( - options: ConstructorParameters[0] & { conversationId?: string }, - ) { - const { conversationId, ...mongoOptions } = options; - super(mongoOptions); - this.conversationId = conversationId; + constructor(options: OwnedCheckpointSaverOptions) { + super(options.serde); + this.conversationId = options.conversationId; + this.repository = + 'repository' in options + ? options.repository + : createMongoCheckpointRepository({ + ...options, + client: options.client as unknown as MongoCheckpointRepositoryOptions['client'], + }); } - private async recordStorage(config: RunnableConfig): Promise { - const namespace = config.configurable?.checkpoint_ns; - const candidate = - ownerOf(config) ?? - (typeof namespace === 'string' ? namespace.match(/^lcg:v2:[0-9a-f]{64}:/)?.[0] : undefined) ?? - config.configurable?.[LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY]; - if (typeof candidate !== 'string' || !/^lcg:v2:[0-9a-f]{64}:$/.test(candidate)) return; - const storage = { - type: 'mongo' as const, - checkpointCollectionName: this.checkpointCollectionName, - checkpointWritesCollectionName: this.checkpointWritesCollectionName, + setup(): Promise { + return this.repository.setup(); + } + + override deleteThread(threadId: string): Promise { + return this.repository.deleteThread(threadId); + } + + private context(config: RunnableConfig): { rootConversationId?: string; storageOwner?: string } { + const storageOwner = config.configurable?.[LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY]; + return { + rootConversationId: this.conversationId, + storageOwner: typeof storageOwner === 'string' ? storageOwner : undefined, }; - await this.db - .collection(CHECKPOINT_STORAGE_COLLECTION) - .updateOne( - { _id: `${candidate}${checkpointStorageKey(storage)}` }, - { $setOnInsert: { owner: candidate, storage } }, - { upsert: true }, - ); } - private async tuple(doc: CheckpointRow, owner: string, legacy = false): Promise { - const namespace = doc.checkpoint_ns.startsWith(owner) - ? doc.checkpoint_ns.slice(owner.length) - : doc.checkpoint_ns; + private async tuple( + doc: SerializedCheckpoint, + requested: CheckpointIdentity, + ): Promise { + const owner = requested.owner; + const legacy = + owner != null && + requested.checkpointId != null && + requested.checkpointId === requested.legacyCheckpointId; const key = { - thread_id: doc.thread_id.startsWith(owner) - ? doc.thread_id.slice(owner.length) - : doc.thread_id, - checkpoint_ns: namespace, - checkpoint_id: doc.checkpoint_id, + thread_id: doc.identity.threadId, + checkpoint_ns: doc.identity.checkpointNs, + checkpoint_id: doc.identity.checkpointId, }; - const rows = await this.db - .collection(this.checkpointWritesCollectionName) - .find({ - checkpoint_id: key.checkpoint_id, - $or: [ - { - thread_id: `${owner}${key.thread_id}`, - checkpoint_ns: `${owner}${namespace}`, - lc_owner: owner, - }, - ...(legacy - ? [{ thread_id: key.thread_id, checkpoint_ns: namespace, lc_owner: { $exists: false } }] - : []), - ], - }) - .toArray(); - const slots = new Map(); - for (const row of rows) { - const slot = JSON.stringify([row.task_id, row.idx]); - if (!slots.has(slot) || row.checkpoint_ns === `${owner}${namespace}`) slots.set(slot, row); - } + const rows = await this.repository.readWrites({ ...requested, ...doc.identity }); const pendingWrites: CheckpointPendingWrite[] = await Promise.all( - [...slots.values()].map( + rows.map( async (row) => [ - row.task_id, + row.taskId, row.channel, - await this.serde.loadsTyped(row.type, row.value.value()), + await this.serde.loadsTyped(row.value.type, row.value.bytes), ] as CheckpointPendingWrite, ), ); @@ -148,22 +114,32 @@ export class OwnedMongoSaver extends MongoDBSaver { config: { configurable: { ...key, - [LIBRECHAT_CHECKPOINT_OWNER_KEY]: owner, - ...(legacy ? { [LIBRECHAT_LEGACY_CHECKPOINT_KEY]: doc.checkpoint_id } : {}), + ...(owner == null ? {} : { [LIBRECHAT_CHECKPOINT_OWNER_KEY]: owner }), + ...(legacy ? { [LIBRECHAT_LEGACY_CHECKPOINT_KEY]: doc.identity.checkpointId } : {}), }, }, - checkpoint: (await this.serde.loadsTyped(doc.type, doc.checkpoint.value())) as Checkpoint, - metadata: (await this.serde.loadsTyped(doc.type, doc.metadata.value())) as CheckpointMetadata, + checkpoint: (await this.serde.loadsTyped( + doc.checkpoint.type, + doc.checkpoint.bytes, + )) as Checkpoint, + metadata: (await this.serde.loadsTyped( + doc.metadata.type, + doc.metadata.bytes, + )) as CheckpointMetadata, pendingWrites, - ...(doc.parent_checkpoint_id == null + ...(doc.parentCheckpointId == null ? {} : { parentConfig: { configurable: { ...key, - checkpoint_id: doc.parent_checkpoint_id, - [LIBRECHAT_CHECKPOINT_OWNER_KEY]: owner, - [LIBRECHAT_LEGACY_CHECKPOINT_KEY]: doc.parent_checkpoint_id, + checkpoint_id: doc.parentCheckpointId, + ...(owner == null + ? {} + : { + [LIBRECHAT_CHECKPOINT_OWNER_KEY]: owner, + [LIBRECHAT_LEGACY_CHECKPOINT_KEY]: doc.parentCheckpointId, + }), }, }, }), @@ -171,35 +147,10 @@ export class OwnedMongoSaver extends MongoDBSaver { } override async getTuple(config: RunnableConfig): Promise { - const owner = ownerOf(config); - if (owner == null) return super.getTuple(config); + if (config.configurable?.thread_id === undefined) return undefined; const key = identity(config); - const checkpoints = this.db.collection(this.checkpointCollectionName); - let doc = await checkpoints - .find({ - ...key, - thread_id: `${owner}${key.thread_id}`, - checkpoint_ns: `${owner}${key.checkpoint_ns}`, - lc_owner: owner, - }) - .sort({ checkpoint_id: -1 }) - .limit(1) - .next(); - if ( - doc == null && - key.checkpoint_id != null && - key.checkpoint_id === config.configurable?.[LIBRECHAT_LEGACY_CHECKPOINT_KEY] - ) { - doc = await checkpoints.findOne({ ...key, lc_owner: { $exists: false } }); - } - return doc == null - ? undefined - : this.tuple( - doc, - owner, - key.checkpoint_id != null && - key.checkpoint_id === config.configurable?.[LIBRECHAT_LEGACY_CHECKPOINT_KEY], - ); + const doc = await this.repository.get(key); + return doc == null ? undefined : this.tuple(doc, key); } override async *list( @@ -207,32 +158,30 @@ export class OwnedMongoSaver extends MongoDBSaver { options?: CheckpointListOptions, ): AsyncGenerator { const owner = ownerOf(config); - if (owner == null) { - yield* super.list(config, options); - return; - } - const key = identity(config); - const query: Filter = { - thread_id: `${owner}${key.thread_id}`, - checkpoint_ns: `${owner}${key.checkpoint_ns}`, - lc_owner: owner, - }; - for (const [name, value] of Object.entries(options?.filter ?? {})) { - if (value !== null && typeof value === 'object') - throw new Error('Checkpoint metadata filters must be primitive'); - query[`metadata_search.${name}`] = value; - } + const key = owner == null ? undefined : identity(config); + const threadId = config.configurable?.thread_id; + const checkpointNs = config.configurable?.checkpoint_ns; const before = options?.before?.configurable?.checkpoint_id; - if (before != null) { - if (typeof before !== 'string') throw new Error('Invalid checkpoint list boundary'); - query.checkpoint_id = { $lt: before }; + if ( + (threadId != null && typeof threadId !== 'string') || + (checkpointNs != null && typeof checkpointNs !== 'string') || + (before != null && typeof before !== 'string') + ) + throw new Error('Invalid checkpoint list boundary'); + for await (const doc of this.repository.list({ + threadId, + checkpointNs: key?.checkpointNs ?? checkpointNs, + owner, + before, + limit: options?.limit, + metadata: options?.filter, + })) { + yield await this.tuple(doc, { + ...(key ?? doc.identity), + checkpointId: undefined, + legacyCheckpointId: undefined, + }); } - let cursor = this.db - .collection(this.checkpointCollectionName) - .find(query) - .sort({ checkpoint_id: -1 }); - if (options?.limit != null) cursor = cursor.limit(options.limit); - for await (const doc of cursor) yield await this.tuple(doc, owner); } override async put( @@ -240,49 +189,34 @@ export class OwnedMongoSaver extends MongoDBSaver { checkpoint: Checkpoint, metadata: CheckpointMetadata, ): Promise { - await this.recordStorage(config); - const owner = ownerOf(config); const key = identity(config); - const parent = key.thread_id === this.conversationId ? undefined : this.conversationId; - if (owner == null && parent == null) return super.put(config, checkpoint, metadata); - const [[type, serializedCheckpoint], [metadataType, serializedMetadata]] = await Promise.all([ + const [[type, bytes], [metadataType, metadataBytes]] = await Promise.all([ this.serde.dumpsTyped(checkpoint), this.serde.dumpsTyped(metadata), ]); if (type !== metadataType) throw new Error('Mismatched checkpoint and metadata types'); const stored = { - thread_id: key.thread_id, - checkpoint_ns: key.checkpoint_ns, - checkpoint_id: checkpoint.id, + threadId: key.threadId, + checkpointNs: key.checkpointNs, + checkpointId: checkpoint.id, }; - await this.db.collection(this.checkpointCollectionName).updateOne( + await this.repository.save( + key, { - ...stored, - ...(owner == null - ? {} - : { - thread_id: `${owner}${key.thread_id}`, - checkpoint_ns: `${owner}${key.checkpoint_ns}`, - lc_owner: owner, - }), + identity: stored, + parentCheckpointId: key.checkpointId, + checkpoint: { type, bytes }, + metadata: { type: metadataType, bytes: metadataBytes }, + metadataSearch: metadata, }, - { - $set: { - ...(parent == null ? {} : { lc_parent_thread: parent }), - parent_checkpoint_id: key.checkpoint_id, - type, - checkpoint: serializedCheckpoint, - metadata: serializedMetadata, - metadata_search: metadata, - }, - ...(this.enableTimestamps ? { $currentDate: { upserted_at: true as const } } : {}), - }, - { upsert: true }, + this.context(config), ); return { configurable: { - ...stored, - ...(owner == null ? {} : { [LIBRECHAT_CHECKPOINT_OWNER_KEY]: owner }), + thread_id: stored.threadId, + checkpoint_ns: stored.checkpointNs, + checkpoint_id: stored.checkpointId, + ...(key.owner == null ? {} : { [LIBRECHAT_CHECKPOINT_OWNER_KEY]: key.owner }), }, }; } @@ -292,71 +226,17 @@ export class OwnedMongoSaver extends MongoDBSaver { writes: PendingWrite[], taskId: string, ): Promise { - await this.recordStorage(config); - const owner = ownerOf(config); const key = identity(config); - const parent = key.thread_id === this.conversationId ? undefined : this.conversationId; - if (owner == null && parent == null) return super.putWrites(config, writes, taskId); - if (key.checkpoint_id == null) throw new Error('Owned writes require a checkpoint id'); - const allSpecial = writes.every(([channel]) => channel in WRITES_IDX_MAP); - const legacy = - owner != null && key.checkpoint_id === config.configurable?.[LIBRECHAT_LEGACY_CHECKPOINT_KEY]; - const existing = - !allSpecial && legacy - ? await this.db - .collection(this.checkpointWritesCollectionName) - .find( - { ...key, task_id: taskId, lc_owner: { $exists: false } }, - { projection: { idx: 1 } }, - ) - .toArray() - : []; - const legacySlots = new Set(existing.map((row) => row.idx)); - const operations = await Promise.all( - writes.map(async ([channel, value], idx) => { - if (legacySlots.has(WRITES_IDX_MAP[channel] ?? idx)) return null; - const [type, serializedValue] = await this.serde.dumpsTyped(value); - const fields = { - channel, - type, - value: serializedValue, - ...(owner == null ? {} : { lc_owner: owner }), - ...(parent == null ? {} : { lc_parent_thread: parent }), - }; - return { - updateOne: { - filter: { - ...key, - ...(owner == null - ? {} - : { - thread_id: `${owner}${key.thread_id}`, - checkpoint_ns: `${owner}${key.checkpoint_ns}`, - lc_owner: owner, - }), - task_id: taskId, - idx: WRITES_IDX_MAP[channel] ?? idx, - }, - update: allSpecial - ? { - $set: fields, - ...(this.enableTimestamps - ? { $currentDate: { upserted_at: true as const } } - : {}), - } - : { - $setOnInsert: { - ...fields, - ...(this.enableTimestamps ? { upserted_at: new Date() } : {}), - }, - }, - upsert: true, - }, - }; + if (key.checkpointId == null) throw new Error('Owned writes require a checkpoint id'); + const serialized = await Promise.all( + writes.map(async ([channel, value], index) => { + const [type, bytes] = await this.serde.dumpsTyped(value); + return { taskId, index: WRITES_IDX_MAP[channel] ?? index, channel, value: { type, bytes } }; }), ); - const pending = operations.filter((operation) => operation != null); - if (pending.length > 0) - await this.db.collection(this.checkpointWritesCollectionName).bulkWrite(pending); + await this.repository.saveWrites({ ...key, checkpointId: key.checkpointId }, serialized, { + ...this.context(config), + replace: writes.every(([channel]) => channel in WRITES_IDX_MAP), + }); } } diff --git a/packages/api/src/agents/checkpoints/storage.ts b/packages/api/src/agents/checkpoints/storage.ts index f09c825592..96cb709a78 100644 --- a/packages/api/src/agents/checkpoints/storage.ts +++ b/packages/api/src/agents/checkpoints/storage.ts @@ -1,128 +1,23 @@ -import mongoose from 'mongoose'; -import { createHash } from 'crypto'; +import { createCheckpointCatalog, checkpointStorageKey } from '@librechat/data-schemas'; import type { TCheckpointerConfig } from 'librechat-data-provider'; +import type { CheckpointCatalog } from '@librechat/data-schemas'; import type { ResolvedCheckpointerConfig } from './config'; -import { checkpointOwnerNamespacePrefix } from '../../stream/checkpoints'; import { resolveCheckpointerConfig } from './config'; -export const CHECKPOINT_STORAGE_COLLECTION = 'agent_checkpoint_stores'; +export { checkpointStorageKey, CHECKPOINT_STORAGE_COLLECTION } from '@librechat/data-schemas'; +export type { CheckpointStorage, CheckpointStorageRecord } from '@librechat/data-schemas'; export const LIBRECHAT_CHECKPOINT_STORAGE_OWNER_KEY = '__librechat_checkpoint_storage_owner'; -export type CheckpointStorage = Omit; -export interface CheckpointStorageRecord { - _id: string; - owner: string; - storage: CheckpointStorage; -} - -export function checkpointStorageKey(storage: CheckpointStorage): string { - const value = JSON.stringify([ - storage.type, - storage.checkpointCollectionName, - storage.checkpointWritesCollectionName, - ]); - return createHash('sha256').update(JSON.stringify(value)).digest('hex'); -} /** Descriptors locate stores; caller ownership and captured identities authorize deletion. */ export async function checkpointStorageConfigs( userId: string, tenantId: string | undefined, cfg?: TCheckpointerConfig, + catalog: CheckpointCatalog = createCheckpointCatalog(), ): Promise { - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) - throw new Error('Checkpoint database is unavailable'); - const owners = (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => - checkpointOwnerNamespacePrefix(userId, tenant), + const configs = (await catalog.configurations(userId, tenantId)).map((storage) => + resolveCheckpointerConfig(storage), ); - const records = await db - .collection(CHECKPOINT_STORAGE_COLLECTION) - .find({ $or: owners.map((owner) => ({ _id: { $regex: `^${owner}` } })) }) - .toArray(); - const configs = records.map(({ storage }) => resolveCheckpointerConfig(storage)); configs.push(resolveCheckpointerConfig(cfg)); return [...new Map(configs.map((storage) => [checkpointStorageKey(storage), storage])).values()]; } - -/** Capture exact owner-qualified identities, including orphan payload during owner-wide deletion. */ -export async function* ownedCheckpointReferences( - userId: string, - tenantId: string | undefined, - conversationIds: readonly string[] | undefined, - storage: CheckpointStorage, -): AsyncGenerator<{ - conversationId: string; - checkpoint: { threadId: string; checkpointNs: string; checkpointId: string }; -}> { - if (storage.type === 'memory') return; - const db = mongoose.connection.db; - if (!db || mongoose.connection.readyState !== 1) - throw new Error('Checkpoint database is unavailable'); - const owners = (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => - checkpointOwnerNamespacePrefix(userId, tenant), - ); - for (let offset = 0; offset < (conversationIds?.length ?? 1); offset += 64) { - const logicalIds = new Map(); - for (const id of conversationIds?.slice(offset, offset + 64) ?? []) { - logicalIds.set(id, id); - for (const owner of owners) logicalIds.set(`${owner}${id}`, id); - } - for (const name of [storage.checkpointCollectionName, storage.checkpointWritesCollectionName]) { - const cursor = db - .collection<{ - thread_id: string; - checkpoint_ns: string; - checkpoint_id: string; - lc_owner?: string; - lc_parent_thread?: string; - }>(name) - .find( - { - ...(conversationIds == null - ? {} - : { - $and: [ - { - $or: [ - { thread_id: { $in: [...logicalIds.keys()] } }, - { lc_parent_thread: { $in: conversationIds.slice(offset, offset + 64) } }, - ], - }, - ], - }), - $or: [ - ...owners.map((owner) => ({ checkpoint_ns: { $regex: `^${owner}` } })), - { lc_owner: { $in: owners } }, - ], - }, - { - projection: { - thread_id: 1, - checkpoint_ns: 1, - checkpoint_id: 1, - lc_owner: 1, - lc_parent_thread: 1, - }, - }, - ); - for await (const row of cursor) { - if (typeof row.checkpoint_id !== 'string' || typeof row.checkpoint_ns !== 'string') - continue; - const logicalThread = - row.lc_owner != null && - owners.includes(row.lc_owner) && - row.thread_id.startsWith(row.lc_owner) - ? row.thread_id.slice(row.lc_owner.length) - : row.thread_id; - yield { - conversationId: row.lc_parent_thread ?? logicalIds.get(row.thread_id) ?? logicalThread, - checkpoint: { - threadId: row.thread_id, - checkpointNs: row.checkpoint_ns, - checkpointId: row.checkpoint_id, - }, - }; - } - } - } -} diff --git a/packages/api/src/stream/checkpoints.ts b/packages/api/src/stream/checkpoints.ts index 3b8b10030d..9e09bd78e7 100644 --- a/packages/api/src/stream/checkpoints.ts +++ b/packages/api/src/stream/checkpoints.ts @@ -1,20 +1,5 @@ -import { createHash, randomUUID } from 'crypto'; - -export const DEFAULT_CHECKPOINT_TTL_SECONDS: number = 24 * 60 * 60; - -const CHECKPOINT_NAMESPACE_PREFIX = 'lcg:v2:'; -function checkpointOwnerHash(userId: string, tenantId?: string): string { - return createHash('sha256') - .update(JSON.stringify([tenantId || null, userId])) - .digest('hex'); -} - -/** Stable prefix derived only from the authenticated owner identity. */ -export function checkpointOwnerNamespacePrefix(userId: string, tenantId?: string): string { - return `${CHECKPOINT_NAMESPACE_PREFIX}${checkpointOwnerHash(userId, tenantId)}:`; -} - -/** Create an opaque, globally unique saver namespace bound to one exact owner. */ -export function createCheckpointNamespace(userId: string, tenantId?: string): string { - return `${checkpointOwnerNamespacePrefix(userId, tenantId)}${randomUUID()}`; -} +export { + DEFAULT_CHECKPOINT_TTL_SECONDS, + checkpointOwnerNamespacePrefix, + createCheckpointNamespace, +} from '@librechat/data-schemas'; diff --git a/packages/data-schemas/src/checkpoints/catalog.ts b/packages/data-schemas/src/checkpoints/catalog.ts new file mode 100644 index 0000000000..38f8ba73a5 --- /dev/null +++ b/packages/data-schemas/src/checkpoints/catalog.ts @@ -0,0 +1,129 @@ +import { createHash } from 'crypto'; +import type { CheckpointStorage, CheckpointCatalog } from './types'; +import { checkpointOwnerNamespacePrefix } from './identity'; +import { checkpointDatabase } from './database'; + +export const CHECKPOINT_STORAGE_COLLECTION = 'agent_checkpoint_stores'; +export interface CheckpointStorageRecord { + _id: string; + owner: string; + storage: CheckpointStorage; +} + +export function checkpointStorageKey(storage: CheckpointStorage): string { + const value = JSON.stringify([ + storage.type, + storage.checkpointCollectionName, + storage.checkpointWritesCollectionName, + ]); + return createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +export function createCheckpointCatalog(): CheckpointCatalog { + return { + async configurations(userId, tenantId) { + const db = checkpointDatabase(); + const owners = (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => + checkpointOwnerNamespacePrefix(userId, tenant), + ); + const records = await db + .collection(CHECKPOINT_STORAGE_COLLECTION) + .find({ $or: owners.map((owner) => ({ _id: { $regex: `^${owner}` } })) }) + .toArray(); + return records.map(({ storage }) => storage); + }, + references: ownedCheckpointReferences, + async forgetDescriptors(userId, tenantId, stores) { + const ids = stores.flatMap((storage) => + (tenantId ? [tenantId, undefined] : [undefined]).map( + (tenant) => + `${checkpointOwnerNamespacePrefix(userId, tenant)}${checkpointStorageKey(storage)}`, + ), + ); + await checkpointDatabase() + .collection(CHECKPOINT_STORAGE_COLLECTION) + .deleteMany({ _id: { $in: [...ids] } }); + }, + }; +} + +/** Capture exact owner-qualified identities, including orphan payload during owner-wide deletion. */ +export async function* ownedCheckpointReferences( + userId: string, + tenantId: string | undefined, + conversationIds: readonly string[] | undefined, + storage: CheckpointStorage, +): AsyncGenerator<{ + conversationId: string; + checkpoint: { threadId: string; checkpointNs: string; checkpointId: string }; +}> { + if (storage.type === 'memory') return; + const db = checkpointDatabase(); + const owners = (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => + checkpointOwnerNamespacePrefix(userId, tenant), + ); + for (let offset = 0; offset < (conversationIds?.length ?? 1); offset += 64) { + const logicalIds = new Map(); + for (const id of conversationIds?.slice(offset, offset + 64) ?? []) { + logicalIds.set(id, id); + for (const owner of owners) logicalIds.set(`${owner}${id}`, id); + } + for (const name of [storage.checkpointCollectionName, storage.checkpointWritesCollectionName]) { + const cursor = db + .collection<{ + thread_id: string; + checkpoint_ns: string; + checkpoint_id: string; + lc_owner?: string; + lc_parent_thread?: string; + }>(name) + .find( + { + ...(conversationIds == null + ? {} + : { + $and: [ + { + $or: [ + { thread_id: { $in: [...logicalIds.keys()] } }, + { lc_parent_thread: { $in: conversationIds.slice(offset, offset + 64) } }, + ], + }, + ], + }), + $or: [ + ...owners.map((owner) => ({ checkpoint_ns: { $regex: `^${owner}` } })), + { lc_owner: { $in: owners } }, + ], + }, + { + projection: { + thread_id: 1, + checkpoint_ns: 1, + checkpoint_id: 1, + lc_owner: 1, + lc_parent_thread: 1, + }, + }, + ); + for await (const row of cursor) { + if (typeof row.checkpoint_id !== 'string' || typeof row.checkpoint_ns !== 'string') + continue; + const logicalThread = + row.lc_owner != null && + owners.includes(row.lc_owner) && + row.thread_id.startsWith(row.lc_owner) + ? row.thread_id.slice(row.lc_owner.length) + : row.thread_id; + yield { + conversationId: row.lc_parent_thread ?? logicalIds.get(row.thread_id) ?? logicalThread, + checkpoint: { + threadId: row.thread_id, + checkpointNs: row.checkpoint_ns, + checkpointId: row.checkpoint_id, + }, + }; + } + } + } +} diff --git a/packages/data-schemas/src/checkpoints/database.ts b/packages/data-schemas/src/checkpoints/database.ts new file mode 100644 index 0000000000..7ba1afa311 --- /dev/null +++ b/packages/data-schemas/src/checkpoints/database.ts @@ -0,0 +1,16 @@ +import mongoose from 'mongoose'; +import type { Db } from 'mongodb'; + +export function isCheckpointDatabaseAvailable(): boolean { + return mongoose.connection.readyState === 1 && mongoose.connection.db != null; +} + +export function checkpointDatabase(): Db { + if (!isCheckpointDatabaseAvailable()) throw new Error('Checkpoint database is unavailable'); + return mongoose.connection.db!; +} + +export function checkpointModelCollectionName(modelName: string, fallback: string): string { + const collection = mongoose.models[modelName]?.collection; + return collection?.name ?? fallback; +} diff --git a/packages/data-schemas/src/checkpoints/identity.ts b/packages/data-schemas/src/checkpoints/identity.ts new file mode 100644 index 0000000000..3b8b10030d --- /dev/null +++ b/packages/data-schemas/src/checkpoints/identity.ts @@ -0,0 +1,20 @@ +import { createHash, randomUUID } from 'crypto'; + +export const DEFAULT_CHECKPOINT_TTL_SECONDS: number = 24 * 60 * 60; + +const CHECKPOINT_NAMESPACE_PREFIX = 'lcg:v2:'; +function checkpointOwnerHash(userId: string, tenantId?: string): string { + return createHash('sha256') + .update(JSON.stringify([tenantId || null, userId])) + .digest('hex'); +} + +/** Stable prefix derived only from the authenticated owner identity. */ +export function checkpointOwnerNamespacePrefix(userId: string, tenantId?: string): string { + return `${CHECKPOINT_NAMESPACE_PREFIX}${checkpointOwnerHash(userId, tenantId)}:`; +} + +/** Create an opaque, globally unique saver namespace bound to one exact owner. */ +export function createCheckpointNamespace(userId: string, tenantId?: string): string { + return `${checkpointOwnerNamespacePrefix(userId, tenantId)}${randomUUID()}`; +} diff --git a/packages/data-schemas/src/checkpoints/indexes.ts b/packages/data-schemas/src/checkpoints/indexes.ts new file mode 100644 index 0000000000..6037db90e9 --- /dev/null +++ b/packages/data-schemas/src/checkpoints/indexes.ts @@ -0,0 +1,45 @@ +import type { CheckpointRepository } from './types'; +import type { IndexBuildOptions } from '../utils'; +import { buildIndexWithRetry, isIndexBuildInProgress } from '../utils'; + +/** Longest wait for a peer's checkpoint index build before the saver proceeds without it. */ +const CHECKPOINT_INDEX_BUILD_DEADLINE_MS = 120_000; + +/** + * `MongoDBSaver.setup()` starts the compound and TTL builds of each collection + * concurrently and reports failures instead of throwing. Amazon DocumentDB + * admits one index build per collection at a time, so there the second build of + * each pair is rejected (code 40333) and the saver would otherwise run with its + * TTL index missing — checkpoints then never expire. `setup()` is idempotent (an + * index that already exists is a no-op), so re-running it lets one more build + * through per pass until every index exists. Every other error is returned for + * the caller to log, exactly as `setup()` reports it — including those reported + * beside a conflict that outlasts the deadline. A rejection of `setup()` itself + * propagates, so the caller keeps its in-process fallback. + */ +export async function setupCheckpointIndexes( + saver: Pick, + options: IndexBuildOptions = {}, +): Promise { + let companions: Error[] = []; + try { + return await buildIndexWithRetry( + async () => { + const errors = await saver.setup(); + const blocked = errors.find(isIndexBuildInProgress); + if (blocked == null) { + return errors; + } + companions = errors.filter((error) => !isIndexBuildInProgress(error)); + throw blocked; + }, + 'MongoDBSaver.setup()', + { peerBuildDeadlineMs: CHECKPOINT_INDEX_BUILD_DEADLINE_MS, ...options }, + ); + } catch (error) { + if (!isIndexBuildInProgress(error)) { + throw error; + } + return [...companions, error instanceof Error ? error : new Error(String(error))]; + } +} diff --git a/packages/data-schemas/src/checkpoints/journal.ts b/packages/data-schemas/src/checkpoints/journal.ts new file mode 100644 index 0000000000..b77a652d9f --- /dev/null +++ b/packages/data-schemas/src/checkpoints/journal.ts @@ -0,0 +1,164 @@ +import mongoose from 'mongoose'; +import { createHash } from 'crypto'; +import type { + CheckpointDeletionJournal, + CheckpointDeletionTarget, + CheckpointReference, + CheckpointStorageConfig, +} from './types'; +import { checkpointDatabase, checkpointModelCollectionName } from './database'; +import { checkpointOwnerNamespacePrefix } from './identity'; +import { checkpointStorageKey } from './catalog'; + +type DeletionRow = Omit & { _id: string }; +const targetFromRow = ({ _id, ...target }: DeletionRow): CheckpointDeletionTarget => ({ + id: _id, + ...target, +}); + +const DELETION_COLLECTION = 'agent_checkpoint_deletions'; + +async function hasPersistence(target: CheckpointDeletionTarget): Promise { + const db = checkpointDatabase(); + const { userId, tenantId, threadId, checkpoint, storage } = target; + const owners = (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => + checkpointOwnerNamespacePrefix(userId, tenant), + ); + const user = mongoose.isValidObjectId(userId) ? new mongoose.Types.ObjectId(userId) : userId; + const siblingCollections = [ + [checkpointModelCollectionName('Conversation', 'conversations'), userId], + [checkpointModelCollectionName('Message', 'messages'), userId], + [checkpointModelCollectionName('ToolCall', 'toolcalls'), user], + [checkpointModelCollectionName('SharedLink', 'sharedlinks'), userId], + ] as const; + const siblings = siblingCollections.map(([name, owner]) => + db + .collection(name) + .findOne({ user: owner, conversationId: threadId }, { projection: { _id: 1 } }), + ); + const payload = + storage.type === 'memory' + ? [] + : [storage.checkpointCollectionName, storage.checkpointWritesCollectionName].map((name) => + db.collection(name).findOne( + { + $or: [ + ...owners.map((owner) => ({ + thread_id: { $in: [threadId, `${owner}${threadId}`] }, + $or: [{ checkpoint_ns: { $regex: `^${owner}` } }, { lc_owner: owner }], + })), + ...(checkpoint == null + ? [] + : [ + { + thread_id: checkpoint.threadId, + checkpoint_ns: checkpoint.checkpointNs, + checkpoint_id: checkpoint.checkpointId, + $or: [{ lc_owner: { $in: owners } }, { lc_owner: { $exists: false } }], + }, + ]), + ], + }, + { projection: { _id: 1 } }, + ), + ); + return (await Promise.all([...siblings, ...payload])).some((row) => row != null); +} + +function deletionPrefixes( + userId: string, + tenantId: string | undefined, + rootConversationId: string | undefined, +): string[] { + const rootHash = createHash('sha256') + .update(JSON.stringify(rootConversationId ?? null)) + .digest('hex'); + return (tenantId ? [tenantId, undefined] : [undefined]).map((tenant) => { + const owner = checkpointOwnerNamespacePrefix(userId, tenant); + return rootConversationId == null ? owner : `${owner}[0-9a-f]{64}:${rootHash}:`; + }); +} + +export function createCheckpointDeletionJournal(): CheckpointDeletionJournal { + const collection = () => checkpointDatabase().collection(DELETION_COLLECTION); + return { + async retained(userId, tenantId, rootConversationId) { + const prefixes = deletionPrefixes(userId, tenantId, rootConversationId); + return ( + await collection() + .find({ $or: prefixes.map((prefix) => ({ _id: { $regex: `^${prefix}` } })) }) + .toArray() + ).map(targetFromRow); + }, + async remember(targets) { + // eslint-disable-next-line no-restricted-syntax -- Native journal collection; receipts are owner-qualified CAS identities. + await collection().bulkWrite( + targets.map(({ id, ...fields }) => ({ + updateOne: { filter: { _id: id }, update: { $set: fields }, upsert: true }, + })), + ); + }, + async acknowledge(targets) { + await collection().deleteMany({ + $or: targets.map(({ id, version }) => ({ _id: id, version })), + }); + }, + async hasPending(userId, tenantId, rootConversationId) { + const prefixes = deletionPrefixes(userId, tenantId, rootConversationId); + return ( + (await collection().findOne( + { $or: prefixes.map((prefix) => ({ _id: { $regex: `^${prefix}` } })) }, + { projection: { _id: 1 } }, + )) != null + ); + }, + async scan(limit, after) { + return ( + await collection() + .find(after == null ? {} : { _id: { $gt: after } }) + .sort({ _id: 1 }) + .limit(limit) + .toArray() + ).map(targetFromRow); + }, + async reclaim(target) { + return (await collection().deleteOne({ _id: target.id, version: target.version })) + .deletedCount; + }, + async hasConversation(userId, threadId) { + return ( + (await checkpointDatabase() + .collection(checkpointModelCollectionName('Conversation', 'conversations')) + .findOne({ user: userId, conversationId: threadId }, { projection: { _id: 1 } })) != null + ); + }, + hasPersistence, + }; +} + +/** Preserve persisted journal keys so mixed-version retries share the same CAS receipts. */ +export function checkpointDeletionTarget( + scope: { userId: string; tenantId?: string; rootConversationId?: string }, + storage: CheckpointStorageConfig, + threadId: string, + version: string, + checkpoint?: CheckpointReference, + proof?: 'owned' | 'historical', +): CheckpointDeletionTarget { + const hash = (value: string | null): string => + createHash('sha256').update(JSON.stringify(value)).digest('hex'); + const prefix = `${checkpointOwnerNamespacePrefix(scope.userId, scope.tenantId)}${checkpointStorageKey(storage)}:${hash(scope.rootConversationId ?? null)}:`; + const evidence = + checkpoint == null + ? '' + : `:${hash(JSON.stringify(proof === 'owned' ? [checkpoint.threadId, checkpoint.checkpointNs, checkpoint.checkpointId] : [checkpoint.checkpointNs, checkpoint.checkpointId]))}`; + return { + id: `${prefix}${hash(threadId)}${evidence}`, + version, + threadId, + userId: scope.userId, + tenantId: scope.tenantId, + storage, + ...(checkpoint == null ? {} : { checkpoint }), + }; +} diff --git a/packages/data-schemas/src/checkpoints/limits.ts b/packages/data-schemas/src/checkpoints/limits.ts new file mode 100644 index 0000000000..48b9e3ba0a --- /dev/null +++ b/packages/data-schemas/src/checkpoints/limits.ts @@ -0,0 +1,50 @@ +/** + * MongoDB's hard per-document ceiling. A checkpoint whose serialized state pushes its + * document past this cannot be stored — the driver throws `BSONObjectTooLarge` (code 10334). + */ +const MAX_BSON_DOCUMENT_BYTES = 16 * 1024 * 1024; + +/** + * Headroom reserved below {@link MAX_BSON_DOCUMENT_BYTES} for a checkpoint document's + * non-state fields (ids, metadata, `metadata_search`, BSON framing). The serialized + * `checkpoint` blob dominates the document; this margin covers everything else so the guard + * rejects before Mongo does — with a legible error instead of a raw driver failure. + */ +const CHECKPOINT_SIZE_HEADROOM_BYTES = 1024 * 1024; + +/** + * Reject a checkpoint whose serialized state exceeds this. The pause is unrecoverable either + * way (the document can't be written), so failing here as a typed {@link CheckpointTooLargeError} + * turns an opaque `BSONObjectTooLarge` crash into an actionable one. + */ +export const CHECKPOINT_HARD_LIMIT_BYTES: number = + MAX_BSON_DOCUMENT_BYTES - CHECKPOINT_SIZE_HEADROOM_BYTES; + +/** + * Warn once a persisted checkpoint crosses this soft threshold (~50% of the ceiling), so a + * conversation's checkpoint growth is visible in logs well before it reaches the hard limit. + */ +export const CHECKPOINT_WARN_BYTES: number = 8 * 1024 * 1024; + +/** + * A durable checkpoint whose serialized state exceeds {@link CHECKPOINT_HARD_LIMIT_BYTES} — more + * than MongoDB can hold in a single document. Thrown BEFORE the doomed write so the run fails + * with a clear, typed message instead of a raw driver `BSONObjectTooLarge`. The checkpoint cannot + * be persisted regardless of how it is handled upstream, so a durable resume is impossible. + */ +export class CheckpointTooLargeError extends Error { + readonly code = 'CHECKPOINT_TOO_LARGE'; + constructor( + readonly bytes: number, + readonly limit: number, + readonly threadId?: string, + ) { + const mb = (n: number): string => (n / 1024 / 1024).toFixed(1); + super( + `Checkpoint state is ${mb(bytes)} MB, over the ${mb(limit)} MB durable limit. ` + + 'This conversation carries too much state to resume — large tool outputs or inlined ' + + 'media are the usual cause. Start a new conversation or reduce context.', + ); + this.name = 'CheckpointTooLargeError'; + } +} diff --git a/packages/data-schemas/src/checkpoints/mongo.spec.ts b/packages/data-schemas/src/checkpoints/mongo.spec.ts new file mode 100644 index 0000000000..cd0a532314 --- /dev/null +++ b/packages/data-schemas/src/checkpoints/mongo.spec.ts @@ -0,0 +1,263 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { CheckpointIdentity, CheckpointRepository, SerializedCheckpoint } from './types'; +import { createCheckpointDeletionJournal, checkpointDeletionTarget } from './journal'; +import { checkpointOwnerNamespacePrefix } from './identity'; +import { createMongoCheckpointRepository } from './mongo'; +import { setupCheckpointIndexes } from './indexes'; + +let server: MongoMemoryServer; +let repository: CheckpointRepository; +let owner: string; +let key: CheckpointIdentity; +const payload = (id: string, parentCheckpointId?: string): SerializedCheckpoint => ({ + identity: { threadId: key.threadId, checkpointNs: key.checkpointNs, checkpointId: id }, + parentCheckpointId, + checkpoint: { type: 'opaque', bytes: Uint8Array.from([0, 255, 42]) }, + metadata: { type: 'opaque', bytes: Uint8Array.from([255, 0]) }, + metadataSearch: { source: 'contract', step: 1 }, +}); +beforeAll(async () => { + server = await MongoMemoryServer.create(); + await mongoose.connect(server.getUri()); +}, 60000); +afterAll(async () => { + await mongoose.disconnect(); + await server.stop(); +}); +beforeEach(() => { + owner = checkpointOwnerNamespacePrefix('owner', 'tenant'); + key = { threadId: 'child', checkpointNs: 'generation[1]', owner }; + repository = createMongoCheckpointRepository({ + checkpointCollectionName: 'cp', + checkpointWritesCollectionName: 'writes', + ttl: 3600, + }); +}); +afterEach(async () => { + await mongoose.connection.dropDatabase(); +}); + +test('round-trips opaque bytes, explicit type tags and lineage with trusted root ownership', async () => { + await repository.save(key, payload('002', '001'), { rootConversationId: 'root' }); + const stored = await repository.get(key); + expect(stored).toEqual(payload('002', '001')); + const row = await mongoose.connection.db!.collection('cp').findOne(); + expect(row).toMatchObject({ + thread_id: `${owner}child`, + lc_owner: owner, + lc_parent_thread: 'root', + upserted_at: expect.any(Date), + }); + expect( + await repository.get({ ...key, owner: checkpointOwnerNamespacePrefix('foreign') }), + ).toBeNull(); +}); + +test('requires an exact historical proof and never falls back to the latest legacy head', async () => { + await repository.save({ ...key, owner: undefined }, payload('001'), {}); + expect(await repository.get(key)).toBeNull(); + expect( + await repository.get({ ...key, checkpointId: '001', legacyCheckpointId: 'other' }), + ).toBeNull(); + expect(await repository.get({ ...key, checkpointId: '001', legacyCheckpointId: '001' })).toEqual( + payload('001'), + ); +}); + +test('preserves legacy insert-only slots and replaces special writes with timestamps', async () => { + const write = { + taskId: 'task', + index: 0, + channel: 'value', + value: { type: 'opaque', bytes: Uint8Array.of(1) }, + }; + await repository.saveWrites({ ...key, owner: undefined, checkpointId: '001' }, [write], { + replace: false, + }); + const owned = { ...key, checkpointId: '001', legacyCheckpointId: '001' }; + await repository.saveWrites( + owned, + [{ ...write, value: { type: 'opaque', bytes: Uint8Array.of(2) } }], + { replace: false }, + ); + expect((await repository.readWrites(owned))[0].value.bytes).toEqual(Uint8Array.of(1)); + await repository.saveWrites( + owned, + [{ ...write, value: { type: 'opaque', bytes: Uint8Array.of(3) } }], + { replace: true, rootConversationId: 'root' }, + ); + expect((await repository.readWrites(owned))[0].value.bytes).toEqual(Uint8Array.of(3)); + const row = await mongoose.connection.db!.collection('writes').findOne({ lc_owner: owner }); + expect(row).toMatchObject({ upserted_at: expect.any(Date), lc_parent_thread: 'root' }); +}); + +test('generation deletion escapes metacharacters and keeps replacement generations', async () => { + const legacy = { ...key, owner: undefined }; + await repository.save(legacy, payload('001'), {}); + await repository.save({ ...legacy, checkpointNs: 'generation1' }, payload('002'), {}); + await repository.save({ ...legacy, checkpointNs: 'generation[1]|nested' }, payload('003'), {}); + await repository.deleteGeneration('child', undefined, 'generation[1]'); + const rows = await mongoose.connection.db!.collection('cp').find().toArray(); + expect(rows.map((row) => row.checkpoint_id)).toEqual(['002']); +}); + +test('rejects ambiguous empty namespaces before deleting either checkpoints or pending writes', async () => { + const checkpoints = mongoose.connection.db!.collection('cp'); + const writes = mongoose.connection.db!.collection('writes'); + const rows = [ + { thread_id: 'child', checkpoint_ns: '', checkpoint_id: '001' }, + { thread_id: 'child', checkpoint_ns: 'nested|child', checkpoint_id: '002' }, + { thread_id: 'child', checkpoint_ns: 'lcg:v2:new-generation', checkpoint_id: '003' }, + ]; + await checkpoints.insertMany(rows); + await writes.insertMany(rows.map((row) => ({ ...row, task_id: 'pending', idx: 0 }))); + await expect(repository.deleteGeneration('child', undefined, '')).rejects.toThrow( + 'captured checkpoint generation', + ); + await expect( + repository.deleteGeneration('child', { + threadId: 'child', + checkpointIds: ['001', '002'], + checkpointNamespace: '', + }), + ).rejects.toThrow('captured checkpoint generation'); + expect(await checkpoints.countDocuments()).toBe(3); + expect(await writes.countDocuments()).toBe(3); + await repository.deleteGeneration('child', { threadId: 'child', checkpointIds: ['001', '002'] }); + expect((await checkpoints.find().toArray()).map((row) => row.checkpoint_id)).toEqual(['003']); + expect((await writes.find().toArray()).map((row) => row.checkpoint_id)).toEqual(['003']); +}); + +test('a captured legacy generation takes precedence over optional namespace hints', async () => { + await repository.save({ ...key, owner: undefined }, payload('001'), {}); + await repository.deleteGeneration('child', { threadId: 'child', checkpointIds: ['001'] }, ''); + expect(await repository.get({ ...key, owner: undefined })).toBeNull(); +}); + +test('owner cleanup includes orphan writes by root but preserves other tenants', async () => { + const write = { + taskId: 'task', + index: 0, + channel: 'value', + value: { type: 'opaque', bytes: Uint8Array.of(1) }, + }; + await repository.saveWrites({ ...key, checkpointId: '001' }, [write], { + replace: false, + rootConversationId: 'root', + }); + const foreign = checkpointOwnerNamespacePrefix('owner', 'another-tenant'); + await repository.saveWrites({ ...key, owner: foreign, checkpointId: '002' }, [write], { + replace: false, + rootConversationId: 'root', + }); + await repository.deleteOwned('owner', 'tenant', ['root']); + expect( + (await mongoose.connection.db!.collection('writes').find().toArray()).map( + (row) => row.lc_owner, + ), + ).toEqual([foreign]); +}); + +test('journal acknowledgement keeps a concurrently refreshed receipt and wire-compatible keys', async () => { + const journal = createCheckpointDeletionJournal(); + const scope = { userId: 'owner', tenantId: 'tenant', rootConversationId: 'root' }; + const store = { + type: 'mongo' as const, + ttlSeconds: 3600, + checkpointCollectionName: 'cp', + checkpointWritesCollectionName: 'writes', + }; + const first = checkpointDeletionTarget( + scope, + store, + 'child', + 'first', + payload('001').identity, + 'owned', + ); + await journal.remember([first]); + await journal.remember([{ ...first, version: 'newer' }]); + await journal.acknowledge([first]); + expect(await journal.retained('owner', 'tenant', 'root')).toEqual([ + { ...first, version: 'newer' }, + ]); + expect(await journal.retained('foreign', 'tenant', 'root')).toEqual([]); + expect(await journal.hasPending('owner', 'tenant', 'root')).toBe(true); + expect(await journal.reclaim(first)).toBe(0); + expect(await journal.reclaim({ ...first, version: 'newer' })).toBe(1); +}); + +test.each(['conversations', 'messages', 'toolcalls', 'sharedlinks'])( + 'retains deletion evidence while %s persistence exists', + async (name) => { + const journal = createCheckpointDeletionJournal(); + const target = checkpointDeletionTarget( + { userId: 'owner' }, + { + type: 'memory', + ttlSeconds: 3600, + checkpointCollectionName: 'cp', + checkpointWritesCollectionName: 'writes', + }, + 'root', + 'version', + ); + expect(await journal.hasPersistence(target)).toBe(false); + await mongoose.connection + .db!.collection(name) + .insertOne({ user: 'owner', conversationId: 'root' }); + expect(await journal.hasPersistence(target)).toBe(true); + }, +); + +test('retries checkpoint index conflicts through the batch deadline and creates TTL indexes', async () => { + const conflict = new mongoose.mongo.MongoServerError({ + code: 40333, + message: 'index build in progress', + }); + const createIndex = mongoose.mongo.Collection.prototype.createIndex; + const builds = new Map(); + jest.spyOn(mongoose.mongo.Collection.prototype, 'createIndex').mockImplementation(function ( + this: mongoose.mongo.Collection, + ...args: Parameters + ) { + const count = (builds.get(this.collectionName) ?? 0) + 1; + builds.set(this.collectionName, count); + if (count === 1) return Promise.reject(conflict); + return createIndex.apply(this, args); + }); + + expect( + await setupCheckpointIndexes(repository, { peerBuildPollMs: 1, peerBuildDeadlineMs: 200 }), + ).toEqual([]); + expect([...builds]).toEqual([ + ['cp', 4], + ['writes', 4], + ]); + for (const name of ['cp', 'writes']) { + expect(await mongoose.connection.db!.collection(name).indexes()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: { upserted_at: 1 }, expireAfterSeconds: 3600 }), + ]), + ); + } +}); + +test('reports every nonretryable index error without retrying individual builds', async () => { + const failure = new Error('index permission denied'); + const createIndex = mongoose.mongo.Collection.prototype.createIndex; + let builds = 0; + jest.spyOn(mongoose.mongo.Collection.prototype, 'createIndex').mockImplementation(function ( + this: mongoose.mongo.Collection, + ...args: Parameters + ) { + if (this.collectionName === 'cp') { + builds++; + return Promise.reject(failure); + } + return createIndex.apply(this, args); + }); + expect(await setupCheckpointIndexes(repository)).toEqual([failure, failure]); + expect(builds).toBe(2); +}); diff --git a/packages/data-schemas/src/checkpoints/mongo.ts b/packages/data-schemas/src/checkpoints/mongo.ts new file mode 100644 index 0000000000..b3e3bb82fe --- /dev/null +++ b/packages/data-schemas/src/checkpoints/mongo.ts @@ -0,0 +1,484 @@ +import mongoose from 'mongoose'; +import type { Filter, Binary, Db, MongoClient } from 'mongodb'; +import type { + CheckpointRepository, + CheckpointIdentity, + CheckpointReference, + CheckpointSelection, + SerializedCheckpoint, + SerializedCheckpointWrite, + CheckpointGeneration, +} from './types'; +import { checkpointStorageKey, CHECKPOINT_STORAGE_COLLECTION } from './catalog'; +import { checkpointOwnerNamespacePrefix } from './identity'; +import { buildIndexWithRetry } from '../utils'; +import logger from '../config/winston'; + +interface CheckpointRow { + thread_id: string; + checkpoint_ns: string; + checkpoint_id: string; + parent_checkpoint_id?: string; + lc_owner?: string; + type: string; + checkpoint: Binary; + metadata: Binary; + metadata_search: Record; +} +interface WriteRow { + thread_id: string; + checkpoint_ns: string; + checkpoint_id: string; + task_id: string; + idx: number; + channel: string; + type: string; + value: Binary; + lc_owner?: string; +} + +export interface MongoCheckpointRepositoryOptions { + client?: Pick; + dbName?: string; + checkpointCollectionName?: string; + checkpointWritesCollectionName?: string; + enableTimestamps?: boolean; + ttl?: number; +} + +function generationNamespaceFilter(namespace: string): string | { $regex: string } { + if (namespace === '') return ''; + return { $regex: `^${namespace.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:[|]|$)` }; +} +function legacyCheckpointStorageFilter() { + return { checkpoint_ns: { $not: /^lcg:v2:/ }, lc_owner: { $exists: false } }; +} +function physical(key: CheckpointIdentity) { + return { + thread_id: `${key.owner ?? ''}${key.threadId}`, + checkpoint_ns: `${key.owner ?? ''}${key.checkpointNs}`, + ...(key.checkpointId == null ? {} : { checkpoint_id: key.checkpointId }), + ...(key.owner == null ? {} : { lc_owner: key.owner }), + }; +} +function decode(row: CheckpointRow, owner?: string): SerializedCheckpoint { + return { + identity: { + threadId: + owner && row.thread_id.startsWith(owner) + ? row.thread_id.slice(owner.length) + : row.thread_id, + checkpointNs: + owner && row.checkpoint_ns.startsWith(owner) + ? row.checkpoint_ns.slice(owner.length) + : row.checkpoint_ns, + checkpointId: row.checkpoint_id, + }, + parentCheckpointId: row.parent_checkpoint_id ?? undefined, + checkpoint: { type: row.type, bytes: row.checkpoint.value() }, + metadata: { type: row.type, bytes: row.metadata.value() }, + metadataSearch: row.metadata_search, + }; +} + +/** Uses the selected app database, or an explicitly supplied client for constructor compatibility. */ +export function createMongoCheckpointRepository( + options: MongoCheckpointRepositoryOptions = {}, +): CheckpointRepository { + const db = options.client?.db(options.dbName) ?? mongoose.connection.db; + if (!db || (!options.client && mongoose.connection.readyState !== 1)) + throw new Error('Checkpoint database is unavailable'); + return new MongoCheckpointRepository(db, options); +} + +class MongoCheckpointRepository implements CheckpointRepository { + private readonly checkpoints: string; + private readonly writes: string; + private readonly timestamps: boolean; + constructor( + private readonly db: Db, + private readonly options: MongoCheckpointRepositoryOptions, + ) { + this.checkpoints = options.checkpointCollectionName ?? 'checkpoints'; + this.writes = options.checkpointWritesCollectionName ?? 'checkpoint_writes'; + this.timestamps = (options.enableTimestamps ?? false) || options.ttl != null; + } + private async recordStorage(key: CheckpointIdentity, storageOwner?: string): Promise { + const candidate = + key.owner ?? key.checkpointNs.match(/^lcg:v2:[0-9a-f]{64}:/)?.[0] ?? storageOwner; + if (typeof candidate !== 'string' || !/^lcg:v2:[0-9a-f]{64}:$/.test(candidate)) return; + const storage = { + type: 'mongo' as const, + checkpointCollectionName: this.checkpoints, + checkpointWritesCollectionName: this.writes, + }; + await this.db + .collection<{ _id: string }>(CHECKPOINT_STORAGE_COLLECTION) + .updateOne( + { _id: `${candidate}${checkpointStorageKey(storage)}` }, + { $setOnInsert: { owner: candidate, storage } }, + { upsert: true }, + ); + } + async setup(): Promise { + // Return individual errors; setupCheckpointIndexes owns the batch retry deadline. + const options = { maxAttempts: 1, peerBuildDeadlineMs: 0 }; + const operations = [ + buildIndexWithRetry( + () => + this.db + .collection(this.checkpoints) + .createIndex( + { thread_id: 1, checkpoint_ns: 1, checkpoint_id: -1 }, + { name: 'thread_ns_checkpoint_idx' }, + ), + `${this.checkpoints}.thread_ns_checkpoint_idx`, + options, + ), + buildIndexWithRetry( + () => + this.db + .collection(this.writes) + .createIndex( + { thread_id: 1, checkpoint_ns: 1, checkpoint_id: 1, task_id: 1, idx: 1 }, + { name: 'thread_ns_checkpoint_task_idx' }, + ), + `${this.writes}.thread_ns_checkpoint_task_idx`, + options, + ), + ]; + if (this.options.ttl != null) + for (const name of [this.checkpoints, this.writes]) + operations.push( + buildIndexWithRetry( + () => + this.db + .collection(name) + .createIndex({ upserted_at: 1 }, { expireAfterSeconds: this.options.ttl }), + `${name}.checkpoint_expiry`, + options, + ), + ); + return (await Promise.allSettled(operations)).flatMap((result) => + result.status === 'rejected' ? [result.reason as Error] : [], + ); + } + async setupOwnershipIndexes(): Promise { + const indexes: Record[] = [ + { checkpoint_ns: 1 }, + { lc_owner: 1, thread_id: 1 }, + { lc_parent_thread: 1 }, + ]; + for (const index of indexes) { + await Promise.all( + [this.checkpoints, this.writes].map(async (name) => { + try { + await buildIndexWithRetry( + () => this.db.collection(name).createIndex(index), + `${name}.checkpoint_ownership`, + ); + } catch (error) { + logger.warn('[checkpointer] Cleanup index unavailable:', error); + } + }), + ); + } + } + async get(key: CheckpointIdentity): Promise { + const checkpoints = this.db.collection(this.checkpoints); + let row = await checkpoints.find(physical(key)).sort({ checkpoint_id: -1 }).limit(1).next(); + if ( + row == null && + key.owner != null && + key.checkpointId != null && + key.checkpointId === key.legacyCheckpointId + ) { + row = await checkpoints.findOne({ + thread_id: key.threadId, + checkpoint_ns: key.checkpointNs, + checkpoint_id: key.checkpointId, + lc_owner: { $exists: false }, + }); + } + return row == null ? null : decode(row, key.owner); + } + async *list(selection: CheckpointSelection): AsyncGenerator { + const query: Filter = { + ...(selection.threadId ? { thread_id: `${selection.owner ?? ''}${selection.threadId}` } : {}), + ...(selection.checkpointNs == null + ? {} + : { checkpoint_ns: `${selection.owner ?? ''}${selection.checkpointNs}` }), + ...(selection.owner == null ? {} : { lc_owner: selection.owner }), + }; + for (const [name, value] of Object.entries(selection.metadata ?? {})) { + if (value !== null && typeof value === 'object') + throw new Error('Checkpoint metadata filters must be primitive'); + query[`metadata_search.${name}`] = value; + } + if (selection.before != null) query.checkpoint_id = { $lt: selection.before }; + let cursor = this.db + .collection(this.checkpoints) + .find(query) + .sort({ checkpoint_id: -1 }); + if (selection.limit != null) cursor = cursor.limit(selection.limit); + for await (const row of cursor) yield decode(row, selection.owner); + } + async readWrites( + key: CheckpointIdentity & { checkpointId: string }, + ): Promise { + const legacy = key.owner != null && key.checkpointId === key.legacyCheckpointId; + const rows = await this.db + .collection(this.writes) + .find({ + checkpoint_id: key.checkpointId, + $or: [ + physical(key), + ...(legacy + ? [ + { + thread_id: key.threadId, + checkpoint_ns: key.checkpointNs, + lc_owner: { $exists: false }, + }, + ] + : []), + ], + }) + .toArray(); + const slots = new Map(); + for (const row of rows) { + const slot = JSON.stringify([row.task_id, row.idx]); + if (!slots.has(slot) || row.checkpoint_ns === `${key.owner ?? ''}${key.checkpointNs}`) + slots.set(slot, row); + } + return [...slots.values()].map((row) => ({ + taskId: row.task_id, + index: row.idx, + channel: row.channel, + value: { type: row.type, bytes: row.value.value() }, + })); + } + async save( + key: CheckpointIdentity, + checkpoint: SerializedCheckpoint, + context: { rootConversationId?: string; storageOwner?: string }, + ): Promise { + await this.recordStorage(key, context.storageOwner); + await this.db.collection(this.checkpoints).updateOne( + physical({ ...key, checkpointId: checkpoint.identity.checkpointId }), + { + $set: { + ...(context.rootConversationId == null || key.threadId === context.rootConversationId + ? {} + : { lc_parent_thread: context.rootConversationId }), + parent_checkpoint_id: checkpoint.parentCheckpointId, + type: checkpoint.checkpoint.type, + checkpoint: checkpoint.checkpoint.bytes, + metadata: checkpoint.metadata.bytes, + metadata_search: checkpoint.metadataSearch, + }, + ...(this.timestamps ? { $currentDate: { upserted_at: true as const } } : {}), + }, + { upsert: true }, + ); + } + async saveWrites( + key: CheckpointIdentity & { checkpointId: string }, + writes: readonly SerializedCheckpointWrite[], + options: { replace: boolean; rootConversationId?: string; storageOwner?: string }, + ): Promise { + await this.recordStorage(key, options.storageOwner); + const legacy = key.owner != null && key.checkpointId === key.legacyCheckpointId; + const existing = + !options.replace && legacy + ? await this.db + .collection(this.writes) + .find( + { + thread_id: key.threadId, + checkpoint_ns: key.checkpointNs, + checkpoint_id: key.checkpointId, + task_id: writes[0]?.taskId, + lc_owner: { $exists: false }, + }, + { projection: { idx: 1 } }, + ) + .toArray() + : []; + const legacySlots = new Set(existing.map((row) => row.idx)); + const operations = writes + .filter((write) => !legacySlots.has(write.index)) + .map((write) => { + const fields = { + channel: write.channel, + type: write.value.type, + value: write.value.bytes, + ...(key.owner == null ? {} : { lc_owner: key.owner }), + ...(options.rootConversationId == null || key.threadId === options.rootConversationId + ? {} + : { lc_parent_thread: options.rootConversationId }), + }; + return { + updateOne: { + filter: { ...physical(key), task_id: write.taskId, idx: write.index }, + update: options.replace + ? { + $set: fields, + ...(this.timestamps ? { $currentDate: { upserted_at: true as const } } : {}), + } + : { + $setOnInsert: { + ...fields, + ...(this.timestamps ? { upserted_at: new Date() } : {}), + }, + }, + upsert: true, + }, + }; + }); + if (operations.length > 0) { + // eslint-disable-next-line no-restricted-syntax -- Native collection; every operation carries its exact checkpoint ownership. + await this.db.collection(this.writes).bulkWrite(operations); + } + } + measureMetadata(metadata: Record): number { + return mongoose.mongo.BSON.calculateObjectSize(metadata); + } + async deleteThread(threadId: string): Promise { + if (typeof threadId !== 'string') throw new Error('Invalid threadId: expected a string'); + await this.db.collection(this.checkpoints).deleteMany({ thread_id: threadId }); + await this.db.collection(this.writes).deleteMany({ thread_id: threadId }); + } + + async captureGeneration(threadId: string, checkpointNamespace?: string): Promise { + const checkpoints = await this.db + .collection<{ checkpoint_id?: string }>(this.checkpoints) + .find( + { + thread_id: threadId, + ...(checkpointNamespace != null + ? { checkpoint_ns: generationNamespaceFilter(checkpointNamespace) } + : legacyCheckpointStorageFilter()), + }, + { projection: { _id: 0, checkpoint_id: 1 } }, + ) + .toArray(); + return checkpoints.reduce((ids, checkpoint) => { + if (typeof checkpoint.checkpoint_id === 'string') { + ids.push(checkpoint.checkpoint_id); + } + return ids; + }, []); + } + + async deleteOwnedScope(threadId: string, checkpointNs: string, owner: string): Promise { + const filter = { + thread_id: `${owner}${threadId}`, + checkpoint_ns: generationNamespaceFilter(`${owner}${checkpointNs}`), + lc_owner: owner, + }; + await Promise.all([ + this.db.collection(this.checkpoints).deleteMany(filter), + this.db.collection(this.writes).deleteMany(filter), + ]); + } + + async deleteOwned( + userId: string, + tenantId: string | undefined, + conversationIds: readonly string[] | undefined, + ): Promise { + const owner = checkpointOwnerNamespacePrefix(userId, tenantId); + const ownership = { $or: [{ checkpoint_ns: { $regex: `^${owner}` } }, { lc_owner: owner }] }; + const ids = conversationIds == null ? undefined : [...new Set(conversationIds)]; + const batchSize = 128; + for (let offset = 0; offset < (ids?.length ?? 1); offset += batchSize) { + const filter = { + ...ownership, + ...(ids && { + $and: [ + { + $or: [ + { + thread_id: { + $in: ids + .slice(offset, offset + batchSize) + .flatMap((id) => [id, `${owner}${id}`]), + }, + }, + { lc_parent_thread: { $in: ids.slice(offset, offset + batchSize) } }, + ], + }, + ], + }), + }; + await this.db.collection(this.checkpoints).deleteMany(filter); + await this.db.collection(this.writes).deleteMany(filter); + } + } + async deleteReference(reference: CheckpointReference, owner?: string): Promise { + const exact = { + thread_id: reference.threadId, + checkpoint_ns: reference.checkpointNs, + checkpoint_id: reference.checkpointId, + ...(owner == null + ? { lc_owner: { $exists: false } } + : { $or: [{ lc_owner: owner }, { lc_owner: { $exists: false } }] }), + }; + await this.db.collection(this.writes).deleteMany(exact); + await this.db.collection(this.checkpoints).deleteOne(exact); + } + async deleteReferences( + references: readonly CheckpointReference[], + owner?: string, + ): Promise { + for (let offset = 0; offset < references.length; offset += 256) { + const exact = { + $and: [ + owner == null + ? { lc_owner: { $exists: false } } + : { $or: [{ lc_owner: owner }, { lc_owner: { $exists: false } }] }, + { + $or: references.slice(offset, offset + 256).map((reference) => ({ + thread_id: reference.threadId, + checkpoint_ns: reference.checkpointNs, + checkpoint_id: reference.checkpointId, + })), + }, + ], + }; + await this.db.collection(this.writes).deleteMany(exact); + await this.db.collection(this.checkpoints).deleteMany(exact); + } + } + async deleteGeneration( + threadId: string, + generation?: CheckpointGeneration, + checkpointNamespace?: string, + ): Promise { + if (generation && (generation.threadId !== threadId || generation.checkpointIds.length === 0)) + return; + if ( + (generation && + Object.prototype.hasOwnProperty.call(generation, 'checkpointNamespace') && + !generation.checkpointNamespace) || + (!generation && checkpointNamespace === '') + ) { + throw new Error( + 'Legacy checkpoint cleanup requires a captured checkpoint generation, not an empty namespace', + ); + } + const namespace = generation ? generation.checkpointNamespace : checkpointNamespace; + const filter = { + thread_id: threadId, + ...(namespace == null + ? legacyCheckpointStorageFilter() + : { checkpoint_ns: generationNamespaceFilter(namespace) }), + ...(generation ? { checkpoint_id: { $in: generation.checkpointIds } } : {}), + }; + await Promise.all([ + this.db.collection(this.checkpoints).deleteMany(filter), + this.db.collection(this.writes).deleteMany(filter), + ]); + } +} diff --git a/packages/data-schemas/src/checkpoints/pruning.ts b/packages/data-schemas/src/checkpoints/pruning.ts new file mode 100644 index 0000000000..01d658a8fc --- /dev/null +++ b/packages/data-schemas/src/checkpoints/pruning.ts @@ -0,0 +1,111 @@ +import type { CheckpointReference, ActorCheckpointPruning } from './types'; +import type { IConversation } from '../types'; +import { checkpointDatabase, checkpointModelCollectionName } from './database'; + +function conversations() { + const db = checkpointDatabase(); + return db.collection & { tenantId?: string | null }>( + checkpointModelCollectionName('Conversation', 'conversations'), + ); +} + +export async function acknowledgeActorPruning( + user: string, + tenantId: string | undefined, + conversationId: string, + reference: CheckpointReference, +): Promise { + await conversations().updateOne( + { user, tenantId: tenantId ?? null, conversationId }, + { $pull: { agentEventActorCleanup: reference } }, + ); +} + +async function pendingActorPruning( + user: string, + tenantId: string | undefined, + conversationId: string, +): Promise { + const conversation = await conversations().findOne( + { user, tenantId: tenantId ?? null, conversationId }, + { projection: { agentEventActorCleanup: 1 } }, + ); + return conversation?.agentEventActorCleanup ?? []; +} + +/** Read hidden historical evidence while the owner-filtered conversation still exists. */ +export async function* historicalActorReferences( + user: string, + tenantId: string | undefined, + conversationIds: readonly string[] | undefined, +): AsyncGenerator { + const ids = conversationIds == null ? undefined : [...new Set(conversationIds)]; + for (let offset = 0; offset < (ids?.length ?? 1); offset += 256) { + const cursor = conversations().find( + { + user, + tenantId: tenantId ? { $in: [tenantId, null] } : null, + subagentThread: { $exists: true }, + ...(ids && { conversationId: { $in: ids.slice(offset, offset + 256) } }), + }, + { + projection: { + conversationId: 1, + 'agentEventActor.checkpoint': 1, + 'agentEventActor.previousCheckpoint': 1, + 'agentEventActorSuspension.suspension.checkpoint': 1, + 'agentEventActorSuspension.suspension.invocation.fork': 1, + 'agentEventActorSuspension.suspension.invocation.base.checkpoint': 1, + 'agentEventActorReconciliations.checkpoint': 1, + agentEventActorCleanup: 1, + }, + }, + ); + for await (const conversation of cursor) { + const suspension = conversation.agentEventActorSuspension?.suspension; + const candidates = [ + conversation.agentEventActor?.checkpoint, + conversation.agentEventActor?.previousCheckpoint, + suspension?.checkpoint, + suspension?.invocation?.fork, + suspension?.invocation?.base?.checkpoint, + ...(conversation.agentEventActorReconciliations ?? []).map((entry) => entry.checkpoint), + ...(conversation.agentEventActorCleanup ?? []), + ]; + const seen = new Set(); + for (const reference of candidates) { + if ( + typeof reference?.checkpointId !== 'string' || + reference.checkpointId.length === 0 || + typeof reference.checkpointNs !== 'string' || + !reference.checkpointNs.startsWith('event-actor/') || + reference.threadId !== conversation.conversationId + ) { + continue; + } + const key = JSON.stringify([ + reference.threadId, + reference.checkpointNs, + reference.checkpointId, + ]); + if (seen.has(key)) { + continue; + } + seen.add(key); + yield { + threadId: reference.threadId, + checkpointNs: reference.checkpointNs, + checkpointId: reference.checkpointId, + }; + } + } + } +} + +export function createActorCheckpointPruning(): ActorCheckpointPruning { + return { + pending: pendingActorPruning, + acknowledge: acknowledgeActorPruning, + historical: historicalActorReferences, + }; +} diff --git a/packages/data-schemas/src/checkpoints/types.ts b/packages/data-schemas/src/checkpoints/types.ts new file mode 100644 index 0000000000..cd531735a1 --- /dev/null +++ b/packages/data-schemas/src/checkpoints/types.ts @@ -0,0 +1,159 @@ +export interface CheckpointReference { + threadId: string; + checkpointNs: string; + checkpointId: string; +} + +export interface CheckpointIdentity { + threadId: string; + checkpointNs: string; + checkpointId?: string; + owner?: string; + /** Exact historical proof, never permission to scan legacy storage. */ + legacyCheckpointId?: string; +} + +export interface CheckpointPayload { + type: string; + bytes: Uint8Array; +} + +export interface SerializedCheckpoint { + identity: CheckpointReference; + parentCheckpointId?: string; + checkpoint: CheckpointPayload; + metadata: CheckpointPayload; + metadataSearch: Record; +} + +export interface SerializedCheckpointWrite { + taskId: string; + index: number; + channel: string; + value: CheckpointPayload; +} + +export interface CheckpointSelection { + threadId?: string; + checkpointNs?: string; + owner?: string; + before?: string; + limit?: number; + metadata?: Record; +} + +export interface CheckpointGeneration { + threadId: string; + checkpointNamespace?: string; + checkpointIds: string[]; +} + +/** Serialized persistence only. Graph configuration, serde and retention belong to the caller. */ +export interface CheckpointRepository { + setup(): Promise; + setupOwnershipIndexes(): Promise; + get(identity: CheckpointIdentity): Promise; + list(selection: CheckpointSelection): AsyncIterable; + readWrites( + identity: CheckpointIdentity & { checkpointId: string }, + ): Promise; + save( + identity: CheckpointIdentity, + checkpoint: SerializedCheckpoint, + context: { rootConversationId?: string; storageOwner?: string }, + ): Promise; + saveWrites( + identity: CheckpointIdentity & { checkpointId: string }, + writes: readonly SerializedCheckpointWrite[], + options: { replace: boolean; rootConversationId?: string; storageOwner?: string }, + ): Promise; + measureMetadata(metadata: Record): number; + deleteThread(threadId: string): Promise; + captureGeneration(threadId: string, checkpointNamespace?: string): Promise; + deleteGeneration( + threadId: string, + generation?: CheckpointGeneration, + checkpointNamespace?: string, + ): Promise; + deleteReference(reference: CheckpointReference, owner?: string): Promise; + deleteReferences(references: readonly CheckpointReference[], owner?: string): Promise; + deleteOwnedScope(threadId: string, checkpointNs: string, owner: string): Promise; + deleteOwned( + userId: string, + tenantId: string | undefined, + conversationIds: readonly string[] | undefined, + ): Promise; +} + +export interface CheckpointStorage { + type: 'mongo' | 'memory'; + checkpointCollectionName: string; + checkpointWritesCollectionName: string; +} + +export interface CheckpointStorageConfig extends CheckpointStorage { + ttlSeconds: number; +} + +export interface CheckpointDeletionTarget { + id: string; + version: string; + threadId: string; + checkpoint?: CheckpointReference; + userId: string; + tenantId?: string; + storage: CheckpointStorageConfig; +} + +export interface CheckpointDeletionJournal { + retained( + userId: string, + tenantId: string | undefined, + rootConversationId: string | undefined, + ): Promise; + remember(targets: readonly CheckpointDeletionTarget[]): Promise; + acknowledge(targets: readonly Pick[]): Promise; + hasPending( + userId: string, + tenantId: string | undefined, + rootConversationId: string | undefined, + ): Promise; + scan(limit: number, after?: string): Promise; + reclaim(target: Pick): Promise; + hasConversation(userId: string, threadId: string): Promise; + hasPersistence(target: CheckpointDeletionTarget): Promise; +} + +export interface CheckpointCatalog { + configurations(userId: string, tenantId: string | undefined): Promise; + references( + userId: string, + tenantId: string | undefined, + conversationIds: readonly string[] | undefined, + storage: CheckpointStorage, + ): AsyncIterable<{ conversationId: string; checkpoint: CheckpointReference }>; + forgetDescriptors( + userId: string, + tenantId: string | undefined, + stores: readonly CheckpointStorage[], + ): Promise; +} + +export interface ActorCheckpointPruning { + pending( + user: string, + tenantId: string | undefined, + conversationId: string, + ): Promise; + acknowledge( + user: string, + tenantId: string | undefined, + conversationId: string, + reference: CheckpointReference, + ): Promise; + historical( + user: string, + tenantId: string | undefined, + conversationIds: readonly string[] | undefined, + ): AsyncIterable; +} diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 9a3381932a..064386cf31 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -184,6 +184,26 @@ export { createBrowserSetupStore, createBrowserSetupBackend } from './methods/br export type * from './types/browserSetup'; export type * from './types/codeEnvironmentSetup'; +export * from './checkpoints/types'; +export * from './checkpoints/identity'; +export { createMongoCheckpointRepository } from './checkpoints/mongo'; +export type { MongoCheckpointRepositoryOptions } from './checkpoints/mongo'; +export { + createCheckpointCatalog, + checkpointStorageKey, + CHECKPOINT_STORAGE_COLLECTION, +} from './checkpoints/catalog'; +export type { CheckpointStorageRecord } from './checkpoints/catalog'; +export { createCheckpointDeletionJournal, checkpointDeletionTarget } from './checkpoints/journal'; +export { createActorCheckpointPruning } from './checkpoints/pruning'; +export { isCheckpointDatabaseAvailable } from './checkpoints/database'; + +export { setupCheckpointIndexes } from './checkpoints/indexes'; +export { + CheckpointTooLargeError, + CHECKPOINT_HARD_LIMIT_BYTES, + CHECKPOINT_WARN_BYTES, +} from './checkpoints/limits'; export * from './postgres/backend'; export * from './postgres/pool'; export * from './postgres/migrations';