mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 21:08:17 +00:00
* 🔒 fix: Isolate Checkpoint Cleanup by Generation Ownership * 🔒 fix: Harden Checkpoint Cleanup Ownership * 🔒 fix: Retain Checkpoint Cleanup Evidence * 🔒 fix: Persist checkpoint cleanup receipts * 🔒 fix: Bound checkpoint cleanup receipt lifecycle * fix: Bind checkpoint cleanup to durable ownership and deletion intent * fix: Bind event actor checkpoint storage to authenticated owners * fix: Resolve exact actor checkpoint ownership before lifecycle cleanup * fix: Preserve actor storage compatibility with durable ownership and prune work * fix: Defer checkpoint erasure until conversation deletion commits * fix: Reclaim orphan actor ownership and batch deletion lookups * fix: bind actor checkpoint ownership to payload rows * fix: isolate actor readers and await account persistence * fix: gate owned suspensions and drain complete account state * fix: unify checkpoint deletion lifecycle and storage authority * fix: Replay checkpoint deletion from durable owner state * fix: retain checkpoint cleanup identity before topology loss * fix: complete checkpoint catalog capture and retirement lifecycle
1435 lines
59 KiB
TypeScript
1435 lines
59 KiB
TypeScript
import mongoose from 'mongoose';
|
||
import { logger } from '@librechat/data-schemas';
|
||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||
import { HumanMessage } from '@librechat/agents/langchain/messages';
|
||
import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb';
|
||
import { emptyCheckpoint, ERROR, INTERRUPT } from '@langchain/langgraph-checkpoint';
|
||
import {
|
||
getAgentCheckpointer,
|
||
hasDurableAgentInterruptCheckpoint,
|
||
captureAgentCheckpointGeneration,
|
||
deleteAgentCheckpoint,
|
||
deleteOwnedAgentCheckpoints,
|
||
forkAgentEventCheckpoint,
|
||
captureAgentEventCheckpoint,
|
||
LazyMongoSaver,
|
||
CheckpointTooLargeError,
|
||
LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
|
||
LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY,
|
||
setupCheckpointIndexes,
|
||
__resetCheckpointerForTests,
|
||
} from './checkpointer';
|
||
import { createCheckpointNamespace } from '../stream/checkpoints';
|
||
|
||
/**
|
||
* Integration tests for the durable Mongo checkpointer seam, against a real
|
||
* (in-memory) MongoDB. The unit spec covers config/selection with no connection;
|
||
* this proves the part that actually matters for correctness — that a checkpoint
|
||
* written for a thread can be read back and that `deleteAgentCheckpoint` truly
|
||
* prunes it (the cross-turn isolation guarantee), scoped to a single thread.
|
||
*/
|
||
|
||
const MONGO_CFG = { type: 'mongo' as const, ttl: 3600 };
|
||
|
||
/** Minimal LangGraph put() args for an empty checkpoint under a thread. */
|
||
function putArgs(threadId: string, checkpointNamespace = '') {
|
||
const config = {
|
||
configurable: { thread_id: threadId, checkpoint_ns: checkpointNamespace },
|
||
};
|
||
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
|
||
return { config, checkpoint: emptyCheckpoint(), metadata };
|
||
}
|
||
|
||
const readConfig = (threadId: string, checkpointNamespace = '') => ({
|
||
configurable: { thread_id: threadId, checkpoint_ns: checkpointNamespace },
|
||
});
|
||
|
||
/**
|
||
* Persist a checkpoint the way LangGraph does on a PAUSE: an interrupt `putWrites` on the
|
||
* `INTERRUPT` channel for the checkpoint id, then `put` of that checkpoint. The lazy saver
|
||
* persists only checkpoints seeded this way (a bare `put` is a clean exit → discarded).
|
||
*/
|
||
async function seedInterruptCheckpoint(
|
||
saver: MongoDBSaver,
|
||
threadId: string,
|
||
checkpointNamespace = '',
|
||
interruptId = 'interrupt-current',
|
||
) {
|
||
const { config, checkpoint, metadata } = putArgs(threadId, checkpointNamespace);
|
||
await saver.putWrites(
|
||
{
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: checkpointNamespace,
|
||
checkpoint_id: checkpoint.id,
|
||
},
|
||
},
|
||
[[INTERRUPT, { id: interruptId, value: 'approve?' }]],
|
||
'task-1',
|
||
);
|
||
await saver.put(config, checkpoint, metadata);
|
||
return checkpoint;
|
||
}
|
||
|
||
let mongoServer: MongoMemoryServer;
|
||
|
||
beforeAll(async () => {
|
||
mongoServer = await MongoMemoryServer.create();
|
||
await mongoose.connect(mongoServer.getUri());
|
||
}, 60000);
|
||
|
||
afterAll(async () => {
|
||
await mongoose.disconnect();
|
||
await mongoServer.stop();
|
||
});
|
||
|
||
beforeEach(() => {
|
||
// Force a fresh saver build (+ setup) against the live connection each test.
|
||
__resetCheckpointerForTests();
|
||
});
|
||
|
||
afterEach(async () => {
|
||
await mongoose.connection.dropDatabase();
|
||
});
|
||
|
||
describe('checkpointer (mongodb-memory-server integration)', () => {
|
||
it('builds a real MongoDBSaver when Mongo is connected', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
expect(saver).toBeDefined();
|
||
// setup() created the checkpoint collection with a TTL index on `upserted_at`.
|
||
const indexes = await mongoose.connection.db!.collection('agent_checkpoints').indexes();
|
||
const ttlIndex = indexes.find((idx) => idx.expireAfterSeconds != null);
|
||
expect(ttlIndex).toBeDefined();
|
||
expect(ttlIndex?.expireAfterSeconds).toBe(3600);
|
||
});
|
||
|
||
it('verifies that an interrupt write has its matching durable checkpoint', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
expect(saver).toBeDefined();
|
||
const missingIdentity = {
|
||
checkpointId: 'missing-checkpoint',
|
||
interruptId: 'interrupt-current',
|
||
};
|
||
await expect(
|
||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, missingIdentity),
|
||
).resolves.toBe(false);
|
||
|
||
const checkpoint = await seedInterruptCheckpoint(saver!, 'verified-pause');
|
||
const identity = {
|
||
checkpointId: checkpoint.id,
|
||
interruptId: 'interrupt-current',
|
||
};
|
||
|
||
await expect(
|
||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, identity),
|
||
).resolves.toBe(true);
|
||
await expect(
|
||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, missingIdentity),
|
||
).resolves.toBe(false);
|
||
await expect(
|
||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, {
|
||
...identity,
|
||
interruptId: 'interrupt-stale',
|
||
}),
|
||
).resolves.toBe(false);
|
||
await mongoose.connection.db!.collection('agent_checkpoints').deleteMany({
|
||
thread_id: 'verified-pause',
|
||
});
|
||
await expect(
|
||
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, identity),
|
||
).resolves.toBe(false);
|
||
});
|
||
|
||
it("uses Mongoose's selected database instead of the MongoClient URI default", async () => {
|
||
await mongoose.disconnect();
|
||
await mongoose.connect(mongoServer.getUri('driver_default'), { dbName: 'active_app' });
|
||
__resetCheckpointerForTests();
|
||
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
expect(saver).toBeDefined();
|
||
await seedInterruptCheckpoint(saver!, 'db-selection-pause');
|
||
|
||
expect(
|
||
await mongoose.connection.db!.collection('agent_checkpoints').countDocuments({
|
||
thread_id: 'db-selection-pause',
|
||
}),
|
||
).toBe(1);
|
||
expect(
|
||
await mongoose.connection
|
||
.getClient()
|
||
.db('driver_default')
|
||
.collection('agent_checkpoints')
|
||
.countDocuments({
|
||
thread_id: 'db-selection-pause',
|
||
}),
|
||
).toBe(0);
|
||
});
|
||
|
||
it('returns undefined for the memory type (SDK MemorySaver fallback) even when connected', async () => {
|
||
expect(await getAgentCheckpointer({ type: 'memory' })).toBeUndefined();
|
||
});
|
||
|
||
it('memoizes one saver per resolved config', async () => {
|
||
const a = await getAgentCheckpointer(MONGO_CFG);
|
||
const b = await getAgentCheckpointer(MONGO_CFG);
|
||
expect(a).toBe(b);
|
||
});
|
||
|
||
it('persists clean event-actor exits and forks only the committed checkpoint', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
|
||
const checkpoint = emptyCheckpoint();
|
||
const config = {
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/base',
|
||
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: 'event-1',
|
||
},
|
||
};
|
||
await saver!.put(config, checkpoint, {
|
||
source: 'input',
|
||
step: -1,
|
||
parents: {},
|
||
});
|
||
|
||
await expect(
|
||
captureAgentEventCheckpoint(threadId, 'event-actor/base', 'event-1', MONGO_CFG),
|
||
).resolves.toEqual({
|
||
threadId,
|
||
checkpointId: checkpoint.id,
|
||
checkpointNs: 'event-actor/base',
|
||
});
|
||
await expect(
|
||
forkAgentEventCheckpoint(
|
||
{ threadId, checkpointId: checkpoint.id, checkpointNs: 'event-actor/base' },
|
||
'event-actor/fork',
|
||
'event-2',
|
||
MONGO_CFG,
|
||
),
|
||
).resolves.toEqual({
|
||
threadId,
|
||
checkpointId: checkpoint.id,
|
||
checkpointNs: 'event-actor/fork',
|
||
});
|
||
expect(
|
||
await mongoose.connection
|
||
.db!.collection('agent_checkpoints')
|
||
.countDocuments({ thread_id: threadId }),
|
||
).toBe(2);
|
||
});
|
||
|
||
it('replaces checkpoint-carried Skill context while preserving the committed base', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
|
||
const checkpoint = emptyCheckpoint();
|
||
checkpoint.channel_values.messages = [
|
||
new HumanMessage({ content: 'ordinary history' }),
|
||
new HumanMessage({
|
||
content: 'old skill body',
|
||
additional_kwargs: { isMeta: true, source: 'skill', skillName: 'analysis' },
|
||
}),
|
||
];
|
||
checkpoint.channel_versions.messages = 1;
|
||
const sourceConfig = {
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/base',
|
||
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: 'event-1',
|
||
},
|
||
};
|
||
await saver!.put(sourceConfig, checkpoint, {
|
||
source: 'input',
|
||
step: -1,
|
||
parents: {},
|
||
});
|
||
|
||
await forkAgentEventCheckpoint(
|
||
{ threadId, checkpointId: checkpoint.id, checkpointNs: 'event-actor/base' },
|
||
'event-actor/fork',
|
||
'event-2',
|
||
MONGO_CFG,
|
||
{
|
||
source: 'skill',
|
||
messages: [
|
||
new HumanMessage({
|
||
content: 'current skill body',
|
||
additional_kwargs: { isMeta: true, source: 'skill', skillName: 'analysis' },
|
||
}),
|
||
],
|
||
},
|
||
);
|
||
|
||
const source = await saver!.getTuple(sourceConfig);
|
||
const fork = await saver!.getTuple({
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
checkpoint_id: checkpoint.id,
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/fork',
|
||
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: 'event-2',
|
||
},
|
||
});
|
||
expect(
|
||
(source?.checkpoint.channel_values.messages as HumanMessage[]).map(
|
||
(message) => message.content,
|
||
),
|
||
).toEqual(['ordinary history', 'old skill body']);
|
||
expect(
|
||
(fork?.checkpoint.channel_values.messages as HumanMessage[]).map(
|
||
(message) => message.content,
|
||
),
|
||
).toEqual(['ordinary history', 'current skill body']);
|
||
});
|
||
|
||
it('warm-continues from a copied actor head without mutating the committed base', async () => {
|
||
const { StateGraph, START, END, Annotation } = await import('@langchain/langgraph');
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
|
||
const State = Annotation.Root({
|
||
events: Annotation<string[]>({
|
||
reducer: (left, right) => [...left, ...right],
|
||
default: () => [],
|
||
}),
|
||
});
|
||
const graph = new StateGraph(State)
|
||
.addNode('observe', (state: { events: string[] }) => ({
|
||
events: [`seen:${state.events[state.events.length - 1]}`],
|
||
}))
|
||
.addEdge(START, 'observe')
|
||
.addEdge('observe', END)
|
||
.compile({ checkpointer: saver as never });
|
||
const config = (checkpointNamespace: string, invocationId: string, checkpointId?: string) => ({
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
|
||
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: invocationId,
|
||
...(checkpointId == null ? {} : { checkpoint_id: checkpointId }),
|
||
},
|
||
durability: 'exit' as const,
|
||
});
|
||
|
||
await graph.invoke({ events: ['event-1'] }, config('event-actor/base', 'event-1'));
|
||
const base = await captureAgentEventCheckpoint(
|
||
threadId,
|
||
'event-actor/base',
|
||
'event-1',
|
||
MONGO_CFG,
|
||
);
|
||
expect(base).not.toBeNull();
|
||
const fork = await forkAgentEventCheckpoint(base!, 'event-actor/fork', 'event-2', MONGO_CFG);
|
||
expect(fork).not.toBeNull();
|
||
|
||
const warm = await graph.invoke(
|
||
{ events: ['event-2'] },
|
||
config('event-actor/fork', 'event-2', fork!.checkpointId),
|
||
);
|
||
expect(warm.events).toEqual(['event-1', 'seen:event-1', 'event-2', 'seen:event-2']);
|
||
const committedBase = await saver!.getTuple({
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
checkpoint_id: base!.checkpointId,
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/base',
|
||
},
|
||
});
|
||
expect(committedBase?.checkpoint.channel_values.events).toEqual(['event-1', 'seen:event-1']);
|
||
});
|
||
|
||
it('cold-rebuilds in a fresh namespace when an invalidated head id is not copied', async () => {
|
||
const { StateGraph, START, END, Annotation } = await import('@langchain/langgraph');
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
|
||
const State = Annotation.Root({
|
||
events: Annotation<string[]>({
|
||
reducer: (left, right) => [...left, ...right],
|
||
default: () => [],
|
||
}),
|
||
});
|
||
const graph = new StateGraph(State)
|
||
.addNode('observe', (state: { events: string[] }) => ({
|
||
events: [`seen:${state.events[state.events.length - 1]}`],
|
||
}))
|
||
.addEdge(START, 'observe')
|
||
.addEdge('observe', END)
|
||
.compile({ checkpointer: saver as never });
|
||
const config = (checkpointNamespace: string, invocationId: string, checkpointId?: string) => ({
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
|
||
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: invocationId,
|
||
...(checkpointId == null ? {} : { checkpoint_id: checkpointId }),
|
||
},
|
||
durability: 'exit' as const,
|
||
});
|
||
|
||
await graph.invoke({ events: ['legacy-before'] }, config('event-actor/base', 'event-1'));
|
||
const base = await captureAgentEventCheckpoint(
|
||
threadId,
|
||
'event-actor/base',
|
||
'event-1',
|
||
MONGO_CFG,
|
||
);
|
||
expect(base).not.toBeNull();
|
||
|
||
const rebuilt = await graph.invoke(
|
||
{ events: ['event-after-legacy'] },
|
||
config('event-actor/cold', 'event-2', base!.checkpointId),
|
||
);
|
||
expect(rebuilt.events).toEqual(['event-after-legacy', 'seen:event-after-legacy']);
|
||
});
|
||
|
||
it('deleteAgentCheckpoint prunes a thread’s persisted checkpoint', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
expect(saver).toBeDefined();
|
||
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
await seedInterruptCheckpoint(saver!, threadId);
|
||
|
||
// The checkpoint is durably readable before pruning…
|
||
expect(await saver!.getTuple(readConfig(threadId))).toBeDefined();
|
||
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG);
|
||
|
||
// …and gone after (so turn N+1 on the same conversationId can't rehydrate it).
|
||
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
|
||
});
|
||
|
||
it('captured legacy cleanup prunes nested graph checkpoints and writes without a thread-wide delete', async () => {
|
||
const { StateGraph, START, END, interrupt, Annotation } = await import('@langchain/langgraph');
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
|
||
const State = Annotation.Root({
|
||
origin: Annotation,
|
||
approved: Annotation,
|
||
});
|
||
const child = new StateGraph(State)
|
||
.addNode('child-gate', (state: { origin?: string }) => ({
|
||
approved: interrupt(`approve-${state.origin}`),
|
||
}))
|
||
.addEdge(START, 'child-gate')
|
||
.addEdge('child-gate', END)
|
||
.compile();
|
||
const graph = new StateGraph(State)
|
||
.addNode('child', child)
|
||
.addEdge(START, 'child')
|
||
.addEdge('child', END)
|
||
.compile({ checkpointer: saver as never });
|
||
const config = {
|
||
configurable: { thread_id: threadId, checkpoint_ns: '' },
|
||
durability: 'exit' as const,
|
||
};
|
||
|
||
await graph.invoke({ origin: 'legacy', approved: null }, config);
|
||
|
||
const db = mongoose.connection.db!;
|
||
const storedNamespaces = await db
|
||
.collection('agent_checkpoints')
|
||
.distinct('checkpoint_ns', { thread_id: threadId });
|
||
expect(storedNamespaces).toContain('');
|
||
expect(storedNamespaces.some((namespace) => namespace.startsWith('child:'))).toBe(true);
|
||
expect(
|
||
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
|
||
).toBeGreaterThan(0);
|
||
|
||
const generation = await captureAgentCheckpointGeneration(threadId, MONGO_CFG, {
|
||
throwOnError: true,
|
||
});
|
||
const replacement = await seedInterruptCheckpoint(saver!, threadId, '2000');
|
||
await expect(
|
||
deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
|
||
throwOnError: true,
|
||
checkpointNamespace: '',
|
||
}),
|
||
).rejects.toThrow('requires a captured checkpoint generation');
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, generation, { throwOnError: true });
|
||
|
||
expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
|
||
1,
|
||
);
|
||
expect(
|
||
await db
|
||
.collection('agent_checkpoints')
|
||
.countDocuments({ thread_id: threadId, checkpoint_ns: { $ne: '2000' } }),
|
||
).toBe(0);
|
||
expect(
|
||
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
|
||
).toBe(1);
|
||
expect(await saver!.getTuple(readConfig(threadId, '2000'))).toMatchObject({
|
||
checkpoint: { id: replacement.id },
|
||
});
|
||
});
|
||
|
||
it('prunes only the targeted thread, leaving other conversations intact', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadA = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const threadB = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
|
||
await seedInterruptCheckpoint(saver!, threadA);
|
||
await seedInterruptCheckpoint(saver!, threadB);
|
||
|
||
await deleteAgentCheckpoint(threadA, MONGO_CFG);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadA))).toBeUndefined();
|
||
expect(await saver!.getTuple(readConfig(threadB))).toBeDefined();
|
||
});
|
||
|
||
it('scheduled namespace capture prunes root and nested writes while preserving a replacement', async () => {
|
||
const db = mongoose.connection.db!;
|
||
const threadId = 'scheduled-owned-run';
|
||
const namespace = createCheckpointNamespace('owner', 'tenant');
|
||
const replacement = createCheckpointNamespace('owner', 'tenant');
|
||
const rows = [namespace, `${namespace}|child`, replacement, ''].map((checkpoint_ns) => ({
|
||
thread_id: threadId,
|
||
checkpoint_ns,
|
||
checkpoint_id: 'same-checkpoint-id',
|
||
}));
|
||
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
|
||
await db.collection(name).insertMany(rows.map((row) => ({ ...row })));
|
||
}
|
||
const capture = await captureAgentCheckpointGeneration(threadId, MONGO_CFG, {
|
||
checkpointNamespace: namespace,
|
||
throwOnError: true,
|
||
});
|
||
expect(capture?.checkpointIds).toContain('same-checkpoint-id');
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, capture, { throwOnError: true });
|
||
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
|
||
expect(await db.collection(name).distinct('checkpoint_ns', { thread_id: threadId })).toEqual(
|
||
expect.arrayContaining([replacement, '']),
|
||
);
|
||
expect(await db.collection(name).countDocuments({ thread_id: threadId })).toBe(2);
|
||
}
|
||
});
|
||
|
||
it('generation-scoped cleanup preserves a replacement checkpoint on the same thread', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const resumed = await seedInterruptCheckpoint(saver!, threadId);
|
||
const generation = await captureAgentCheckpointGeneration(threadId, MONGO_CFG);
|
||
|
||
const replacement = await seedInterruptCheckpoint(saver!, threadId);
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, generation);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId))).toMatchObject({
|
||
checkpoint: { id: replacement.id },
|
||
});
|
||
expect(
|
||
await saver!.getTuple({
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: '',
|
||
checkpoint_id: resumed.id,
|
||
},
|
||
}),
|
||
).toBeUndefined();
|
||
expect(
|
||
await mongoose.connection
|
||
.db!.collection('agent_checkpoint_writes')
|
||
.countDocuments({ thread_id: threadId, checkpoint_id: replacement.id }),
|
||
).toBe(1);
|
||
expect(
|
||
await mongoose.connection
|
||
.db!.collection('agent_checkpoint_writes')
|
||
.countDocuments({ thread_id: threadId, checkpoint_id: resumed.id }),
|
||
).toBe(0);
|
||
});
|
||
|
||
it('legacy capture and cleanup preserve ordinary owned payload even with matching checkpoint IDs', async () => {
|
||
const saver = (await getAgentCheckpointer(MONGO_CFG))!;
|
||
const threadId = 'legacy-owner-collision';
|
||
const legacy = await seedInterruptCheckpoint(saver, threadId);
|
||
const ownedNamespace = createCheckpointNamespace('other-owner', 'tenant');
|
||
const db = mongoose.connection.db!;
|
||
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
|
||
const row = await db.collection(name).findOne({ thread_id: threadId });
|
||
const { _id, ...payload } = row!;
|
||
await db.collection(name).insertOne({ ...payload, checkpoint_ns: ownedNamespace });
|
||
await db
|
||
.collection(name)
|
||
.insertOne({ ...payload, checkpoint_id: 'only-owned', checkpoint_ns: ownedNamespace });
|
||
}
|
||
const captured = await captureAgentCheckpointGeneration(threadId, MONGO_CFG);
|
||
expect(captured.checkpointIds).toEqual([legacy.id]);
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, captured, { throwOnError: true });
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, { throwOnError: true });
|
||
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
|
||
const rows = await db.collection(name).find({ thread_id: threadId }).toArray();
|
||
expect(rows).toHaveLength(2);
|
||
expect(rows.every((row) => row.checkpoint_ns === ownedNamespace)).toBe(true);
|
||
}
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
|
||
checkpointNamespace: ownedNamespace,
|
||
throwOnError: true,
|
||
});
|
||
expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
|
||
0,
|
||
);
|
||
expect(
|
||
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
|
||
).toBe(0);
|
||
});
|
||
|
||
it('isolates a fresh generation from a predecessor checkpoint written after replacement', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const predecessorNamespace = '1000';
|
||
const replacementNamespace = '2000';
|
||
|
||
await seedInterruptCheckpoint(saver!, threadId, predecessorNamespace);
|
||
// Fresh B prunes only its own namespace before graph construction. A is
|
||
// still running remotely and writes another interrupt after that barrier.
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
|
||
checkpointNamespace: replacementNamespace,
|
||
});
|
||
const latePredecessor = await seedInterruptCheckpoint(saver!, threadId, predecessorNamespace);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId, replacementNamespace))).toBeUndefined();
|
||
const replacement = await seedInterruptCheckpoint(saver!, threadId, replacementNamespace);
|
||
expect(await saver!.getTuple(readConfig(threadId, replacementNamespace))).toMatchObject({
|
||
checkpoint: { id: replacement.id },
|
||
});
|
||
expect(await saver!.getTuple(readConfig(threadId, predecessorNamespace))).toMatchObject({
|
||
checkpoint: { id: latePredecessor.id },
|
||
});
|
||
});
|
||
|
||
it('adapts LangGraph root namespaces and prevents a late predecessor pause from hydrating into its replacement', async () => {
|
||
const { StateGraph, START, END, interrupt, Annotation, Command } = await import(
|
||
'@langchain/langgraph'
|
||
);
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
|
||
const State = Annotation.Root({
|
||
origin: Annotation,
|
||
approved: Annotation,
|
||
});
|
||
const graph = new StateGraph(State)
|
||
.addNode('gate', (state: { origin?: string }) => ({
|
||
approved: interrupt(`approve-${state.origin}`),
|
||
}))
|
||
.addEdge(START, 'gate')
|
||
.addEdge('gate', END)
|
||
.compile({ checkpointer: saver as never });
|
||
|
||
const scopedConfig = (generation: string, graphNamespace = '') => ({
|
||
configurable: {
|
||
thread_id: threadId,
|
||
checkpoint_ns: graphNamespace,
|
||
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: generation,
|
||
},
|
||
durability: 'exit' as const,
|
||
});
|
||
// Deliberately nonempty on the initial root calls: PregelLoop.initialize
|
||
// resets these to ''. The ordinary client wiring uses the normalized
|
||
// configs below on both fresh and resume calls.
|
||
const replacementStartConfig = scopedConfig('generation-b', 'caller-root-generation-b');
|
||
const predecessorStartConfig = scopedConfig('generation-a', 'caller-root-generation-a');
|
||
const replacementConfig = scopedConfig('generation-b');
|
||
const predecessorConfig = scopedConfig('generation-a');
|
||
|
||
// B has started and paused. A's remote owner then lands a late pause on
|
||
// the same conversation thread; physical namespaces must keep it out of B.
|
||
await graph.invoke({ origin: 'B', approved: null }, replacementStartConfig);
|
||
await graph.invoke({ origin: 'A', approved: null }, predecessorStartConfig);
|
||
|
||
const storedNamespaces = await mongoose.connection
|
||
.db!.collection('agent_checkpoints')
|
||
.distinct('checkpoint_ns', { thread_id: threadId });
|
||
expect(new Set(storedNamespaces)).toEqual(new Set(['generation-a', 'generation-b']));
|
||
expect(storedNamespaces).not.toContain('');
|
||
expect(storedNamespaces).not.toContain('caller-root-generation-a');
|
||
expect(storedNamespaces).not.toContain('caller-root-generation-b');
|
||
|
||
const replacementState = await graph.getState(replacementConfig);
|
||
expect(replacementState.values).toMatchObject({ origin: 'B' });
|
||
const listedReplacementCheckpoints = [];
|
||
for await (const tuple of saver!.list(replacementConfig)) {
|
||
listedReplacementCheckpoints.push(tuple);
|
||
}
|
||
expect(listedReplacementCheckpoints.length).toBeGreaterThan(0);
|
||
expect(
|
||
listedReplacementCheckpoints.every(
|
||
(tuple) =>
|
||
tuple.config.configurable?.checkpoint_ns === '' &&
|
||
tuple.config.configurable?.[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY] === 'generation-b',
|
||
),
|
||
).toBe(true);
|
||
const replacementResult = await graph.invoke(
|
||
new Command({ resume: 'YES-B' }),
|
||
replacementConfig,
|
||
);
|
||
expect(replacementResult).toMatchObject({ origin: 'B', approved: 'YES-B' });
|
||
|
||
// Terminal B cleanup removes its whole namespace, including any write
|
||
// committed after an earlier snapshot, while late A remains resumable.
|
||
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
|
||
checkpointNamespace: 'generation-b',
|
||
});
|
||
expect(await graph.getState(replacementConfig)).toMatchObject({ next: [] });
|
||
expect(await graph.getState(predecessorConfig)).toMatchObject({
|
||
values: expect.objectContaining({ origin: 'A' }),
|
||
next: expect.arrayContaining(['gate']),
|
||
});
|
||
});
|
||
|
||
it('deleteAgentCheckpoint is a no-op for an undefined threadId', async () => {
|
||
await expect(deleteAgentCheckpoint(undefined, MONGO_CFG)).resolves.toBeUndefined();
|
||
});
|
||
|
||
it('bulk-deletes only an owned generation scope when thread IDs collide', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `collision-${new mongoose.Types.ObjectId().toString()}`;
|
||
const ownedNamespace = createCheckpointNamespace('owner', 'tenant');
|
||
await seedInterruptCheckpoint(saver!, threadId, ownedNamespace);
|
||
await seedInterruptCheckpoint(saver!, threadId, `${ownedNamespace}|subgraph`);
|
||
await seedInterruptCheckpoint(saver!, threadId, 'foreign-generation');
|
||
await seedInterruptCheckpoint(saver!, threadId, '');
|
||
|
||
await deleteOwnedAgentCheckpoints('owner', 'tenant', [threadId, threadId], MONGO_CFG);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId, ownedNamespace))).toBeUndefined();
|
||
expect(
|
||
await saver!.getTuple(readConfig(threadId, `${ownedNamespace}|subgraph`)),
|
||
).toBeUndefined();
|
||
expect(await saver!.getTuple(readConfig(threadId, 'foreign-generation'))).toBeDefined();
|
||
expect(await saver!.getTuple(readConfig(threadId, ''))).toBeDefined();
|
||
const db = mongoose.connection.db!;
|
||
expect(
|
||
await db.collection('agent_checkpoint_writes').countDocuments({
|
||
thread_id: threadId,
|
||
checkpoint_ns: { $in: [ownedNamespace, `${ownedNamespace}|subgraph`] },
|
||
}),
|
||
).toBe(0);
|
||
expect(
|
||
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
|
||
).toBe(2);
|
||
});
|
||
|
||
it('propagates a bulk checkpoint deletion failure for retry', async () => {
|
||
const deletion = jest
|
||
.spyOn(mongoose.mongo.Collection.prototype, 'deleteMany')
|
||
.mockRejectedValueOnce(new Error('checkpoint deletion unavailable'));
|
||
try {
|
||
await expect(
|
||
deleteOwnedAgentCheckpoints('owner', 'tenant', ['conversation-1'], MONGO_CFG),
|
||
).rejects.toThrow('checkpoint deletion unavailable');
|
||
} finally {
|
||
deletion.mockRestore();
|
||
}
|
||
});
|
||
});
|
||
|
||
describe('LazyMongoSaver (lazy persistence — mongodb-memory-server)', () => {
|
||
it('does NOT persist a clean-exit checkpoint (a bare put with no pending writes)', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { config, checkpoint, metadata } = putArgs(threadId);
|
||
|
||
// A non-paused run's exit put — no preceding putWrites.
|
||
await saver!.put(config, checkpoint, metadata);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
|
||
const count = await mongoose.connection
|
||
.db!.collection('agent_checkpoints')
|
||
.countDocuments({ thread_id: threadId });
|
||
expect(count).toBe(0);
|
||
});
|
||
|
||
it('persists a checkpoint anchored by a NON-interrupt write (delta-channel safety)', async () => {
|
||
// K1/K3: a delta-channel graph can putWrites on a checkpoint that an interrupt
|
||
// checkpoint then depends on — even without the __interrupt__ marker. A write on a
|
||
// real (non-`__`-prefixed) channel must anchor its checkpoint so resume can walk the chain.
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { config, checkpoint, metadata } = putArgs(threadId);
|
||
await saver!.putWrites(
|
||
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
|
||
[['some_delta_channel', { msgs: ['delta'] }]],
|
||
'task-1',
|
||
);
|
||
await saver!.put(config, checkpoint, metadata);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId))).toBeDefined();
|
||
});
|
||
|
||
it('does NOT persist an error-only checkpoint OR its write row (failed non-paused turn — no leak)', async () => {
|
||
// A turn that throws before any pause records a pending write on the `__error__`
|
||
// bookkeeping channel, then a `put` (probe-confirmed against @langchain/langgraph). That
|
||
// checkpoint is never HITL-resumable, so the lazy saver must leave NOTHING durable: the
|
||
// bookkeeping batch is PARKED (not forwarded) until the checkpoint's fate is known, and the
|
||
// discarding `put` drops both — no dead checkpoint, no orphan row in the writes collection.
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { config, checkpoint, metadata } = putArgs(threadId);
|
||
await saver!.putWrites(
|
||
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
|
||
[[ERROR, 'boom']], // '__error__' — bookkeeping channel, not resumable
|
||
'task-1',
|
||
);
|
||
await saver!.put(config, checkpoint, metadata);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
|
||
const db = mongoose.connection.db!;
|
||
expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
|
||
0,
|
||
);
|
||
expect(
|
||
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
|
||
).toBe(0);
|
||
});
|
||
|
||
it('preserves bookkeeping writes on a RETAINED checkpoint, in either arrival order', async () => {
|
||
// Codex M2 (probe-confirmed): when one Send-sibling interrupts, siblings that completed
|
||
// without state updates are recorded as `__no_writes__` pending writes on the SAME retained
|
||
// checkpoint. Those markers must persist — dropping them makes LangGraph re-execute the
|
||
// completed siblings on resume (duplicated side effects). The saver parks bookkeeping
|
||
// batches until the checkpoint is anchored, so both orders must end durable.
|
||
const NO_WRITES = '__no_writes__'; // langgraph constants.NO_WRITES (runner marker)
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
|
||
const runOrder = async (bookkeepingFirst: boolean) => {
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { config, checkpoint, metadata } = putArgs(threadId);
|
||
const writeCfg = {
|
||
configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id },
|
||
};
|
||
const bookkeeping = () => saver!.putWrites(writeCfg, [[NO_WRITES, null]], 'task-sibling');
|
||
const anchor = () => saver!.putWrites(writeCfg, [[INTERRUPT, 'approve?']], 'task-gate');
|
||
if (bookkeepingFirst) {
|
||
await bookkeeping();
|
||
await anchor();
|
||
} else {
|
||
await anchor();
|
||
await bookkeeping();
|
||
}
|
||
await saver!.put(config, checkpoint, metadata);
|
||
|
||
const tuple = await saver!.getTuple(readConfig(threadId));
|
||
expect(tuple).toBeDefined();
|
||
const channels = (tuple?.pendingWrites ?? []).map((w) => w[1]);
|
||
expect(channels).toContain(INTERRUPT);
|
||
expect(channels).toContain(NO_WRITES);
|
||
};
|
||
|
||
await runOrder(true); // parked, then flushed by the anchoring batch
|
||
await runOrder(false); // forwarded directly (checkpoint already anchored)
|
||
});
|
||
|
||
it('un-anchors a checkpoint whose putWrites failed (no phantom pause persisted)', async () => {
|
||
// A transient Mongo failure while writing the interrupt batch must not leave a persisted
|
||
// checkpoint with MISSING pending writes: LangGraph dispatches the matching `put`
|
||
// concurrently with `putWrites` (probe-confirmed), so the failed batch's anchor is removed
|
||
// on rejection and that `put` discards the checkpoint instead of saving a phantom pause.
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { config, checkpoint, metadata } = putArgs(threadId);
|
||
|
||
const spy = jest
|
||
.spyOn(MongoDBSaver.prototype, 'putWrites')
|
||
.mockRejectedValueOnce(new Error('transient mongo failure'));
|
||
try {
|
||
await expect(
|
||
saver!.putWrites(
|
||
{
|
||
configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id },
|
||
},
|
||
[[INTERRUPT, 'approve?']],
|
||
'task-1',
|
||
),
|
||
).rejects.toThrow('transient mongo failure');
|
||
} finally {
|
||
spy.mockRestore();
|
||
}
|
||
|
||
// The `put` LangGraph issues for that checkpoint finds no anchor → discarded.
|
||
await saver!.put(config, checkpoint, metadata);
|
||
|
||
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
|
||
const count = await mongoose.connection
|
||
.db!.collection('agent_checkpoints')
|
||
.countDocuments({ thread_id: threadId });
|
||
expect(count).toBe(0);
|
||
});
|
||
|
||
it('persists an interrupt checkpoint and carries its __interrupt__ pending write', async () => {
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
await seedInterruptCheckpoint(saver!, threadId);
|
||
|
||
const tuple = await saver!.getTuple(readConfig(threadId));
|
||
expect(tuple).toBeDefined();
|
||
// pendingWrites entries are [taskId, channel, value]; the interrupt is on INTERRUPT.
|
||
expect((tuple?.pendingWrites ?? []).some((w) => w[1] === INTERRUPT)).toBe(true);
|
||
});
|
||
|
||
it('end-to-end: a real graph writes nothing on a clean run, a checkpoint on interrupt', async () => {
|
||
const { StateGraph, START, END, interrupt, Annotation } = await import('@langchain/langgraph');
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const coll = mongoose.connection.db!.collection('agent_checkpoints');
|
||
|
||
const State = Annotation.Root({ x: Annotation });
|
||
const build = (withInterrupt: boolean) =>
|
||
new StateGraph(State)
|
||
.addNode('a', () => {
|
||
if (withInterrupt) {
|
||
interrupt('approve?');
|
||
}
|
||
return { x: 'done' };
|
||
})
|
||
.addEdge(START, 'a')
|
||
.addEdge('a', END)
|
||
// version skew between checkpoint-mongodb's BaseCheckpointSaver and langgraph's.
|
||
.compile({ checkpointer: saver as never });
|
||
|
||
// Clean run → nothing persisted.
|
||
const tClean = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
await build(false).invoke(
|
||
{ x: 'start' },
|
||
{ configurable: { thread_id: tClean }, durability: 'exit' },
|
||
);
|
||
expect(await coll.countDocuments({ thread_id: tClean })).toBe(0);
|
||
|
||
// Interrupt run → a checkpoint is persisted and the graph reports a pending pause.
|
||
const tPause = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const pauseGraph = build(true);
|
||
await pauseGraph.invoke(
|
||
{ x: 'start' },
|
||
{ configurable: { thread_id: tPause }, durability: 'exit' },
|
||
);
|
||
expect(await coll.countDocuments({ thread_id: tPause })).toBeGreaterThan(0);
|
||
const state = await pauseGraph.getState({ configurable: { thread_id: tPause } });
|
||
expect(state.next.length).toBeGreaterThan(0); // the interrupted node is still pending → resumable
|
||
});
|
||
|
||
it('end-to-end: a real graph that THROWS before pausing persists no checkpoint', async () => {
|
||
// F2: a failed non-paused turn records an `__error__` pending write + a put. The lazy
|
||
// saver must discard it so a conversation that errors (and is never retried) leaves nothing
|
||
// durable behind — the clean-path prune that used to catch this was removed.
|
||
const { StateGraph, START, END, Annotation } = await import('@langchain/langgraph');
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
const coll = mongoose.connection.db!.collection('agent_checkpoints');
|
||
|
||
const State = Annotation.Root({ x: Annotation });
|
||
const boomGraph = new StateGraph(State)
|
||
.addNode('a', () => {
|
||
throw new Error('boom');
|
||
})
|
||
.addEdge(START, 'a')
|
||
.addEdge('a', END)
|
||
.compile({ checkpointer: saver as never });
|
||
|
||
const tErr = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
await expect(
|
||
boomGraph.invoke({ x: 'start' }, { configurable: { thread_id: tErr }, durability: 'exit' }),
|
||
).rejects.toThrow('boom');
|
||
|
||
// Nothing durable: neither the checkpoint nor an orphan row in the writes collection.
|
||
expect(await coll.countDocuments({ thread_id: tErr })).toBe(0);
|
||
const writesColl = mongoose.connection.db!.collection('agent_checkpoint_writes');
|
||
expect(await writesColl.countDocuments({ thread_id: tErr })).toBe(0);
|
||
});
|
||
|
||
it('end-to-end: an interrupt persists, then resumes to completion with the approval value', async () => {
|
||
// Guards the `putWrites` change: the `__interrupt__` write must still be forwarded (it is
|
||
// resumable) so a paused run rehydrates and the resume value flows in. Mirrors the real HITL
|
||
// round-trip across a fresh `invoke` on the same thread_id.
|
||
const { StateGraph, START, END, interrupt, Annotation, Command } = await import(
|
||
'@langchain/langgraph'
|
||
);
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
|
||
const State = Annotation.Root({ approved: Annotation });
|
||
const graph = new StateGraph(State)
|
||
.addNode('gate', () => ({ approved: interrupt('approve?') }))
|
||
.addNode('done', () => ({}))
|
||
.addEdge(START, 'gate')
|
||
.addEdge('gate', 'done')
|
||
.addEdge('done', END)
|
||
.compile({ checkpointer: saver as never });
|
||
|
||
const thread = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const cfg = { configurable: { thread_id: thread }, durability: 'exit' as const };
|
||
|
||
// Pause at the interrupt.
|
||
await graph.invoke({ approved: null }, cfg);
|
||
const paused = await graph.getState(cfg);
|
||
expect(paused.next.length).toBeGreaterThan(0);
|
||
|
||
// Resume with the approval — the paused run rehydrates from the durable interrupt checkpoint.
|
||
const out = await graph.invoke(new Command({ resume: 'YES' }), cfg);
|
||
expect(out.approved).toBe('YES');
|
||
});
|
||
|
||
it('end-to-end: completed Send-siblings are NOT re-executed after a pause (no duplicate side effects)', async () => {
|
||
// Codex M2 end-to-end: a Send fan-out where 'b' pauses for approval while 'a'/'c' complete
|
||
// with side effects but NO state writes (→ `__no_writes__` markers on the retained
|
||
// checkpoint). On resume — through a REBUILT graph, as resume.js rebuilds the Run — the
|
||
// completed siblings must not run again. Before the buffering fix this probe measured
|
||
// effects {a:2, c:2}: the dropped markers made LangGraph re-execute both siblings.
|
||
const { StateGraph, START, END, interrupt, Annotation, Send, Command } = await import(
|
||
'@langchain/langgraph'
|
||
);
|
||
const saver = await getAgentCheckpointer(MONGO_CFG);
|
||
|
||
const effects: Record<string, number> = {};
|
||
const State = Annotation.Root({
|
||
items: Annotation({ reducer: (a: string[], b: string[]) => (a ?? []).concat(b ?? []) }),
|
||
results: Annotation({ reducer: (a: string[], b: string[]) => (a ?? []).concat(b ?? []) }),
|
||
});
|
||
const build = () =>
|
||
new StateGraph(State)
|
||
.addNode('fan', () => ({}))
|
||
.addNode('work', (s: { item?: string }) => {
|
||
if (s.item === 'b') {
|
||
return { results: [`b:${interrupt('approve b?')}`] };
|
||
}
|
||
effects[s.item!] = (effects[s.item!] ?? 0) + 1; // side effect, no state update
|
||
return {}; // no channel writes → langgraph records a `__no_writes__` marker
|
||
})
|
||
.addConditionalEdges(
|
||
'fan',
|
||
(s) => s.items.map((i: string) => new Send('work', { item: i })),
|
||
['work'],
|
||
)
|
||
.addEdge(START, 'fan')
|
||
.addEdge('work', END)
|
||
.compile({ checkpointer: saver as never });
|
||
|
||
const thread = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const cfg = { configurable: { thread_id: thread }, durability: 'exit' as const };
|
||
|
||
await build().invoke({ items: ['a', 'b', 'c'], results: [] }, cfg); // pauses on 'b'
|
||
expect(effects).toEqual({ a: 1, c: 1 });
|
||
|
||
// Resume on a REBUILT graph sharing the durable saver (mirrors resume.js).
|
||
const out = await build().invoke(new Command({ resume: 'YES' }), cfg);
|
||
expect(out.results).toEqual(['b:YES']);
|
||
expect(effects).toEqual({ a: 1, c: 1 }); // siblings did NOT re-execute
|
||
});
|
||
});
|
||
|
||
describe('LazyMongoSaver checkpoint size guard (mongodb-memory-server integration)', () => {
|
||
// Guards the single-document ceiling: a checkpoint embeds the whole message history, so a
|
||
// large conversation can serialize past MongoDB's 16 MB limit. The guard measures the
|
||
// serialized state on the persist path, WARNS past a soft threshold, and REJECTS past a hard
|
||
// limit BEFORE the write — a typed CheckpointTooLargeError instead of a raw BSONObjectTooLarge.
|
||
// Thresholds are shrunk here so payloads stay tiny while exercising the real serde + Mongo.
|
||
const clientForSaver = () =>
|
||
// mongoose vends the live MongoClient; the driver type resolves to a different `mongodb` copy
|
||
// than checkpoint-mongodb's, so the cast mirrors buildMongoSaver in checkpointer.ts.
|
||
mongoose.connection.getClient() as unknown as ConstructorParameters<
|
||
typeof MongoDBSaver
|
||
>[0]['client'];
|
||
|
||
const makeSaver = (overrides?: { warnBytes?: number; hardLimitBytes?: number }) =>
|
||
new LazyMongoSaver({
|
||
client: clientForSaver(),
|
||
checkpointCollectionName: 'agent_checkpoints',
|
||
checkpointWritesCollectionName: 'agent_checkpoint_writes',
|
||
ttl: 3600,
|
||
...overrides,
|
||
});
|
||
|
||
/** Seed an interrupt anchor for a checkpoint whose serialized state is inflated to ~`payloadBytes`. */
|
||
async function seedSizedInterrupt(saver: LazyMongoSaver, threadId: string, payloadBytes: number) {
|
||
const checkpoint = emptyCheckpoint();
|
||
checkpoint.channel_values = { messages: 'x'.repeat(payloadBytes) };
|
||
await saver.putWrites(
|
||
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
|
||
[[INTERRUPT, 'approve?']],
|
||
'task-1',
|
||
);
|
||
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
|
||
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
|
||
return { checkpoint, config, metadata };
|
||
}
|
||
|
||
afterEach(() => {
|
||
jest.restoreAllMocks();
|
||
});
|
||
|
||
it('persists a checkpoint under the soft threshold', async () => {
|
||
const saver = makeSaver({ warnBytes: 5_000, hardLimitBytes: 50_000 });
|
||
await saver.setup();
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 100);
|
||
|
||
await saver.put(config, checkpoint, metadata);
|
||
|
||
expect(await saver.getTuple(readConfig(threadId))).toBeDefined();
|
||
});
|
||
|
||
it('persists but WARNS when a checkpoint crosses the soft threshold', async () => {
|
||
const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
|
||
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 50_000 });
|
||
await saver.setup();
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 2_000);
|
||
|
||
await saver.put(config, checkpoint, metadata);
|
||
|
||
expect(await saver.getTuple(readConfig(threadId))).toBeDefined();
|
||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('soft threshold'));
|
||
});
|
||
|
||
it('REFUSES to persist and throws CheckpointTooLargeError over the hard limit', async () => {
|
||
jest.spyOn(logger, 'error').mockImplementation(() => logger);
|
||
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 2_000 });
|
||
await saver.setup();
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 10_000);
|
||
|
||
await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf(
|
||
CheckpointTooLargeError,
|
||
);
|
||
|
||
// Nothing durable was written for that thread — getTuple reads the checkpoint document.
|
||
expect(await saver.getTuple(readConfig(threadId))).toBeUndefined();
|
||
expect(
|
||
await mongoose.connection.db!.collection('agent_checkpoints').countDocuments({
|
||
thread_id: threadId,
|
||
}),
|
||
).toBe(0);
|
||
});
|
||
|
||
it('counts metadata toward the ceiling — refuses when the checkpoint is under but metadata pushes over', async () => {
|
||
// MongoDBSaver stores the serialized checkpoint AND metadata in one document, so a
|
||
// just-under-limit checkpoint with large metadata still overflows. The guard must catch it
|
||
// as a typed CheckpointTooLargeError, not let it fall through to a raw BSONObjectTooLarge.
|
||
jest.spyOn(logger, 'error').mockImplementation(() => logger);
|
||
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 2_000 });
|
||
await saver.setup();
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
|
||
const checkpoint = emptyCheckpoint();
|
||
checkpoint.channel_values = { messages: 'x'.repeat(400) }; // checkpoint alone well UNDER 2 KB
|
||
await saver.putWrites(
|
||
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
|
||
[[INTERRUPT, 'approve?']],
|
||
'task-1',
|
||
);
|
||
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
|
||
// Metadata alone pushes checkpoint + metadata over the 2 KB hard limit.
|
||
const metadata = {
|
||
source: 'input' as const,
|
||
step: -1,
|
||
writes: { payload: 'y'.repeat(4_000) },
|
||
parents: {},
|
||
};
|
||
|
||
await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf(
|
||
CheckpointTooLargeError,
|
||
);
|
||
expect(await saver.getTuple(readConfig(threadId))).toBeUndefined();
|
||
});
|
||
|
||
it('counts metadata_search (the raw metadata copy) — refuses when serialized-only would pass', async () => {
|
||
// MongoDBSaver stores `metadata_search: metadata` — the WHOLE raw metadata a
|
||
// SECOND time in the same document. Sized so checkpoint + serialized metadata is
|
||
// UNDER the limit but adding the raw metadata_search copy pushes it over: the guard
|
||
// must count that copy, else the doc slips the preflight and overflows on the wire.
|
||
jest.spyOn(logger, 'error').mockImplementation(() => logger);
|
||
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 6_000 });
|
||
await saver.setup();
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
|
||
const checkpoint = emptyCheckpoint();
|
||
checkpoint.channel_values = { messages: 'x'.repeat(200) };
|
||
await saver.putWrites(
|
||
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
|
||
[[INTERRUPT, 'approve?']],
|
||
'task-1',
|
||
);
|
||
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
|
||
// ~3 KB metadata: checkpoint + serialized metadata (~3.2 KB) is under 6 KB, but the
|
||
// raw metadata_search copy (~another 3 KB) pushes checkpoint+metadata+metadata_search
|
||
// over 6 KB. The pre-fix guard (checkpoint + serialized metadata only) would pass here.
|
||
const metadata = {
|
||
source: 'loop' as const,
|
||
step: 1,
|
||
writes: { payload: 'y'.repeat(3_000) },
|
||
parents: {},
|
||
};
|
||
|
||
await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf(
|
||
CheckpointTooLargeError,
|
||
);
|
||
expect(await saver.getTuple(readConfig(threadId))).toBeUndefined();
|
||
});
|
||
|
||
it('flushes bookkeeping parked during the size-serialization window (not dropped on resume)', async () => {
|
||
// `put` consumes the write anchor, then AWAITS `assertCheckpointFitsDocument` (serialization).
|
||
// A bookkeeping-only putWrites dispatched in that window sees no anchor and no persisted
|
||
// marker, so it parks — and must be flushed once the checkpoint persists, or a resume
|
||
// re-executes the completed sibling.
|
||
const NO_WRITES = '__no_writes__';
|
||
const saver = makeSaver({ warnBytes: 5_000, hardLimitBytes: 50_000 });
|
||
await saver.setup();
|
||
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
|
||
const checkpoint = emptyCheckpoint();
|
||
const writeCfg = {
|
||
configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id },
|
||
};
|
||
await saver.putWrites(writeCfg, [[INTERRUPT, 'approve?']], 'task-gate');
|
||
|
||
// Pause the checkpoint serialization inside `put` so the bookkeeping batch lands mid-window.
|
||
const serde = (saver as unknown as { serde: { dumpsTyped: (v: unknown) => unknown } }).serde;
|
||
const realDumps = serde.dumpsTyped.bind(serde);
|
||
let releaseGate!: () => void;
|
||
const gate = new Promise<void>((resolve) => {
|
||
releaseGate = resolve;
|
||
});
|
||
let paused = false;
|
||
jest.spyOn(serde, 'dumpsTyped').mockImplementation(async (value: unknown) => {
|
||
if (!paused) {
|
||
paused = true;
|
||
await gate; // hold inside assertCheckpointFitsDocument (anchor consumed, not yet persisted)
|
||
}
|
||
return realDumps(value);
|
||
});
|
||
|
||
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
|
||
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
|
||
const putPromise = saver.put(config, checkpoint, metadata); // suspends at the gate
|
||
await saver.putWrites(writeCfg, [[NO_WRITES, null]], 'task-sibling'); // parks in the window
|
||
releaseGate();
|
||
await putPromise;
|
||
|
||
const tuple = await saver.getTuple(readConfig(threadId));
|
||
const channels = (tuple?.pendingWrites ?? []).map((w) => w[1]);
|
||
expect(channels).toContain(INTERRUPT);
|
||
expect(channels).toContain(NO_WRITES); // flushed, not dropped
|
||
});
|
||
});
|
||
|
||
describe('setupCheckpointIndexes on a single-index-build engine (mongodb-memory-server)', () => {
|
||
const CHECKPOINTS = 'single_build_checkpoints';
|
||
const WRITES = 'single_build_checkpoint_writes';
|
||
/** Amazon DocumentDB: "Existing index build in progress on the same collection." */
|
||
const INDEX_BUILD_ALREADY_IN_PROGRESS = 40333;
|
||
|
||
const clientForSaver = () =>
|
||
mongoose.connection.getClient() as unknown as ConstructorParameters<
|
||
typeof MongoDBSaver
|
||
>[0]['client'];
|
||
|
||
/** Bound to Mongoose's database the way `buildMongoSaver` binds it, so the
|
||
* index assertions below read the collections the saver actually built. */
|
||
const makeSaver = () =>
|
||
new LazyMongoSaver({
|
||
client: clientForSaver(),
|
||
dbName: mongoose.connection.db?.databaseName,
|
||
checkpointCollectionName: CHECKPOINTS,
|
||
checkpointWritesCollectionName: WRITES,
|
||
ttl: 3600,
|
||
});
|
||
|
||
function indexBuildInProgressError(): Error {
|
||
return new mongoose.mongo.MongoServerError({
|
||
ok: 0,
|
||
code: INDEX_BUILD_ALREADY_IN_PROGRESS,
|
||
errmsg:
|
||
'Existing index build in progress on the same collection. Collection is limited to a single index build at a time.',
|
||
});
|
||
}
|
||
|
||
interface IndexBuild {
|
||
collectionName: string;
|
||
indexName: string;
|
||
build: () => Promise<string>;
|
||
}
|
||
|
||
/**
|
||
* Routes every `createIndex` through `handler`, naming the index the way the
|
||
* server would. Patched on the driver prototype because the saver fetches a
|
||
* fresh `Collection` handle per call.
|
||
*/
|
||
function patchCreateIndex(handler: (build: IndexBuild) => Promise<string>): () => void {
|
||
const prototype = mongoose.mongo.Collection.prototype;
|
||
const createIndex = prototype.createIndex;
|
||
prototype.createIndex = function (this: mongoose.mongo.Collection, spec, options) {
|
||
const indexName =
|
||
options?.name ??
|
||
Object.entries(spec)
|
||
.map(([field, direction]) => `${field}_${direction}`)
|
||
.join('_');
|
||
return handler({
|
||
collectionName: this.collectionName,
|
||
indexName,
|
||
build: () => createIndex.call(this, spec, options),
|
||
});
|
||
};
|
||
return () => {
|
||
prototype.createIndex = createIndex;
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Turns the in-memory MongoDB into a single-index-build engine for the two
|
||
* checkpoint collections: a build that arrives while another is in flight on
|
||
* the same collection is rejected the way DocumentDB rejects it, instead of
|
||
* being serialized the way MongoDB does. Re-creating an index that already
|
||
* exists starts no build on either engine, so it always passes through.
|
||
*/
|
||
function enforceSingleIndexBuild(): { rejected: () => number; restore: () => void } {
|
||
const inFlight = new Set<string>();
|
||
const built = new Set<string>();
|
||
let rejected = 0;
|
||
const restore = patchCreateIndex(async ({ collectionName, indexName, build }) => {
|
||
const key = `${collectionName}:${indexName}`;
|
||
if ((collectionName !== CHECKPOINTS && collectionName !== WRITES) || built.has(key)) {
|
||
return build();
|
||
}
|
||
if (inFlight.has(collectionName)) {
|
||
rejected += 1;
|
||
throw indexBuildInProgressError();
|
||
}
|
||
inFlight.add(collectionName);
|
||
try {
|
||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||
const name = await build();
|
||
built.add(key);
|
||
return name;
|
||
} finally {
|
||
inFlight.delete(collectionName);
|
||
}
|
||
});
|
||
return { rejected: () => rejected, restore };
|
||
}
|
||
|
||
async function indexNames(collectionName: string): Promise<string[]> {
|
||
const indexes = await mongoose.connection.db!.collection(collectionName).indexes();
|
||
return indexes.map((index) => index.name ?? '');
|
||
}
|
||
|
||
it('models the engine: a raw setup() loses one build per collection', async () => {
|
||
const engine = enforceSingleIndexBuild();
|
||
try {
|
||
const errors = await makeSaver().setup();
|
||
|
||
expect(errors.map((error) => (error as { code?: number }).code)).toEqual([
|
||
INDEX_BUILD_ALREADY_IN_PROGRESS,
|
||
INDEX_BUILD_ALREADY_IN_PROGRESS,
|
||
]);
|
||
expect(engine.rejected()).toBe(2);
|
||
} finally {
|
||
engine.restore();
|
||
}
|
||
});
|
||
|
||
it('re-runs setup() until every checkpoint index exists', async () => {
|
||
const engine = enforceSingleIndexBuild();
|
||
try {
|
||
await expect(
|
||
setupCheckpointIndexes(makeSaver(), { peerBuildPollMs: 1, peerBuildDeadlineMs: 5_000 }),
|
||
).resolves.toEqual([]);
|
||
|
||
expect(engine.rejected()).toBe(2);
|
||
expect(await indexNames(CHECKPOINTS)).toEqual(
|
||
expect.arrayContaining(['thread_ns_checkpoint_idx', 'upserted_at_1']),
|
||
);
|
||
expect(await indexNames(WRITES)).toEqual(
|
||
expect.arrayContaining(['thread_ns_checkpoint_task_idx', 'upserted_at_1']),
|
||
);
|
||
} finally {
|
||
engine.restore();
|
||
}
|
||
});
|
||
|
||
it('returns errors that are not a concurrent build without re-running setup()', async () => {
|
||
const restore = patchCreateIndex(async ({ collectionName, build }) => {
|
||
if (collectionName !== WRITES) {
|
||
return build();
|
||
}
|
||
throw new mongoose.mongo.MongoServerError({
|
||
ok: 0,
|
||
code: 67,
|
||
errmsg: 'CannotCreateIndex: bad index spec',
|
||
});
|
||
});
|
||
try {
|
||
const saver = makeSaver();
|
||
const setup = jest.spyOn(saver, 'setup');
|
||
|
||
const errors = await setupCheckpointIndexes(saver, {
|
||
peerBuildPollMs: 1,
|
||
peerBuildDeadlineMs: 5_000,
|
||
});
|
||
|
||
expect(errors.map((error) => error.message)).toEqual([
|
||
'CannotCreateIndex: bad index spec',
|
||
'CannotCreateIndex: bad index spec',
|
||
]);
|
||
expect(setup).toHaveBeenCalledTimes(1);
|
||
} finally {
|
||
restore();
|
||
}
|
||
});
|
||
|
||
it('proceeds with the conflict reported once the peer-build deadline passes', async () => {
|
||
const restore = patchCreateIndex(async ({ collectionName, build }) => {
|
||
if (collectionName !== CHECKPOINTS) {
|
||
return build();
|
||
}
|
||
throw indexBuildInProgressError();
|
||
});
|
||
try {
|
||
const errors = await setupCheckpointIndexes(makeSaver(), {
|
||
peerBuildPollMs: 1,
|
||
peerBuildDeadlineMs: 20,
|
||
});
|
||
|
||
expect(errors.map((error) => (error as { code?: number }).code)).toEqual([
|
||
INDEX_BUILD_ALREADY_IN_PROGRESS,
|
||
]);
|
||
} finally {
|
||
restore();
|
||
}
|
||
});
|
||
|
||
it('keeps the failures reported beside a conflict that outlasts the deadline', async () => {
|
||
const restore = patchCreateIndex(async ({ collectionName }) => {
|
||
if (collectionName === CHECKPOINTS) {
|
||
throw indexBuildInProgressError();
|
||
}
|
||
throw new mongoose.mongo.MongoServerError({
|
||
ok: 0,
|
||
code: 67,
|
||
errmsg: 'CannotCreateIndex: bad index spec',
|
||
});
|
||
});
|
||
try {
|
||
const errors = await setupCheckpointIndexes(makeSaver(), {
|
||
peerBuildPollMs: 1,
|
||
peerBuildDeadlineMs: 20,
|
||
});
|
||
|
||
expect(errors.map((error) => (error as { code?: number }).code).sort()).toEqual([
|
||
INDEX_BUILD_ALREADY_IN_PROGRESS,
|
||
67,
|
||
67,
|
||
]);
|
||
} finally {
|
||
restore();
|
||
}
|
||
});
|
||
|
||
it('propagates a setup() rejection that is not a build conflict', async () => {
|
||
/** The driver validates collection names server-side and `setup()` settles
|
||
* every build with `Promise.allSettled`, so there is no rejection path to
|
||
* trigger for real; a rejecting stand-in covers the contract that such a
|
||
* rejection reaches the caller's in-process fallback instead of being
|
||
* reported as an index error. */
|
||
const saver = makeSaver();
|
||
saver.setup = () => Promise.reject(new Error('client closed'));
|
||
|
||
await expect(setupCheckpointIndexes(saver, { peerBuildPollMs: 1 })).rejects.toThrow(
|
||
'client closed',
|
||
);
|
||
});
|
||
});
|