📨 feat: Coordinate Channel Delivery Through Responses (#17609)

* 📨 feat: Coordinate Channel Delivery Through Responses

* fix: Enforce Channel Credential Retirement and Live Trace Delivery

* test: derive artifact policy fixtures from validated schema

* Enforce Current Channel Credentials and Recover Worker Initialization

* Complete Channel Capability and Durable Reconciliation Boundaries
This commit is contained in:
Dustin Healy 2026-11-10 21:04:02 -08:00 • committed by GitHub
parent aae13f6817
commit c907dbf346
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 5773 additions and 12 deletions

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,629 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import { channelConfigSchema } from 'librechat-data-provider';
import {
createModels,
createMethods,
createChannelRunStorage,
createResponseStorage,
runAsSystem,
tenantStorage,
} from '@librechat/data-schemas';
import type { ChannelRunRecord } from '@librechat/data-schemas';
import type { ChannelCleanupDependencies } from './cleanup';
import { createResponseRetirement } from '~/agents/responses/retirement';
import { createChannelResponseRetirement } from './responses';
import { createChannelExecutionAdapter } from './execution';
import { createChannelCleanupRuntime } from './cleanup';
import * as recovery from '~/agents/recovery';
let mongo: MongoMemoryReplSet;
let db: ReturnType<typeof createMethods>;
let storage: ReturnType<typeof createChannelRunStorage>;
let deps: ChannelCleanupDependencies;
let runtime: ReturnType<typeof createChannelCleanupRuntime>;
let retired: ChannelRunRecord[];
let allowed: boolean;
let enabled: boolean;
let stoppedResult: 'stopped' | 'pending';
let clock: number;
const tenantId = 'cleanup-tenant';
const scope = <T>(work: () => Promise<T>) => tenantStorage.run({ tenantId }, work);
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } });
await mongoose.connect(mongo.getUri());
createModels(mongoose);
db = createMethods(mongoose);
storage = createChannelRunStorage(mongoose);
await Promise.all([mongoose.models.ChannelRun.init(), mongoose.models.ChannelLane.init()]);
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
beforeEach(async () => {
await runAsSystem(() =>
Promise.all(
[
'ChannelRun',
'ChannelLane',
'AgentChannel',
'ChannelIdentity',
'ChannelScope',
'ResponseScope',
'StoredResponse',
].map((name) => mongoose.models[name].deleteMany({})),
),
);
clock = Date.now();
allowed = true;
enabled = true;
retired = [];
stoppedResult = 'stopped';
await scope(() =>
db.createAgentChannel({
channelId: 'channel',
tenantId,
agentId: 'agent',
createdBy: 'user-a',
type: 'slack',
credentialMode: 'per_user',
status: 'active',
revision: 'v1',
encryptedBotToken: 'not-a-real-token',
slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' },
}),
);
await scope(() =>
mongoose.models.ChannelIdentity.create({
tenantId,
linkId: 'link',
revision: 'link-v1',
state: 'active',
userId: 'user-a',
ticketHash: 'ticket',
ticketExpiresAtMs: clock + 60000,
expiresAtMs: clock + 600000,
identity: {
tenantId,
channelId: 'channel',
channelRevision: 'v1',
teamId: 'T1',
externalUserId: 'U1',
agentId: 'agent',
},
}),
);
deps = {
storage,
isCredentialCurrent: async () => true,
channels: db,
now: () => clock,
getConfig: async () =>
channelConfigSchema.parse({
enabled,
cleanup: { intervalMs: 60_000, maxIdleIntervalMs: 60_000, batchSize: 2 },
}),
resolvePrincipal: async () =>
allowed
? { userId: 'user-a', channelRevision: 'v1', linkId: 'link', linkRevision: 'link-v1' }
: null,
retireForAuthorityLoss: async (run) => {
expect(tenantStorage.getStore()?.tenantId).toBe(
run.tenantId === '_' ? undefined : run.tenantId,
);
retired.push(run);
return stoppedResult;
},
deleteApiKey: async () => true,
onError: jest.fn(),
};
runtime = createChannelCleanupRuntime(deps);
});
afterEach(async () => {
await runtime.stop();
});
async function admit(eventId = 'event') {
return scope(() =>
storage.admit(
{
tenantId,
channelId: 'channel',
agentId: 'agent',
principal: {
userId: 'user-a',
channelRevision: 'v1',
linkId: 'link',
linkRevision: 'link-v1',
},
admission: {
eventId,
externalChannelId: 'C1',
externalThreadKey: 'C1:1',
externalMessageKey: `C1:${eventId}`,
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
input: 'private input',
},
},
clock,
),
);
}
async function restart() {
await runtime.stop();
runtime = createChannelCleanupRuntime(deps);
await runtime.initialize();
}
it('does not start by construction, initializes once, and leaves authorized active work untouched', async () => {
const { run } = await admit();
expect(retired).toHaveLength(0);
await Promise.all([runtime.initialize(), runtime.initialize()]);
expect(retired).toHaveLength(0);
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
state: 'queued',
leaseEpoch: 0,
cancellationRequested: false,
});
await runtime.stop();
allowed = false;
await runtime.initialize();
expect(retired).toHaveLength(0);
});
it.each(['revoked', 'disabled', 'feature-off', 'orphan'] as const)(
'cleans %s work without a gateway or any credential bundle',
async (reason) => {
const { run } = await admit();
await scope(() => storage.claim(tenantId, 'channel', 'absent-gateway', clock, 60_000));
if (reason === 'revoked') allowed = false;
if (reason === 'disabled')
await scope(() => db.updateAgentChannel('channel', 'user-a', { status: 'disabled' }));
if (reason === 'feature-off') enabled = false;
if (reason === 'orphan')
await scope(() => mongoose.models.AgentChannel.deleteOne({ channelId: 'channel' }));
const bundles = jest.spyOn(db, 'getAgentChannel');
await runtime.initialize();
expect(bundles).not.toHaveBeenCalled();
expect(retired).toHaveLength(1);
expect(retired[0]).toMatchObject({ runId: run.runId, tenantId, principal: run.principal });
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
state: 'cancelled',
errorCode: 'channel_authority_changed',
});
expect(deps.onError).not.toHaveBeenCalled();
},
);
it('defers deletion while cleanup is unknown and completes after restart and exact lease expiry', async () => {
const { run } = await admit();
await scope(() => db.updateAgentChannel('channel', 'user-a', { status: 'deleting' }));
expect(await scope(() => db.deleteAgentChannel('channel', 'user-a'))).toEqual({
status: 'pending_drain',
});
stoppedResult = 'pending';
await runtime.initialize();
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toMatchObject({
status: 'deleting',
});
await restart();
expect(retired).toHaveLength(1);
clock += 60_001;
stoppedResult = 'stopped';
await restart();
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toBeNull();
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
state: 'cancelled',
});
});
it('recovers deleting parents with no remaining runs but never bypasses a failed legacy key revocation', async () => {
await scope(() => db.updateAgentChannel('channel', 'user-a', { apiKeyId: 'legacy-key-id' }));
await scope(() => db.beginAgentChannelDeletion('channel', 'user-a'));
deps.deleteApiKey = async () => {
throw new Error('key store unavailable');
};
await runtime.initialize();
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toMatchObject({
status: 'deleting',
});
deps.deleteApiKey = async () => true;
await restart();
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toBeNull();
});
it('does not convert authority lookup outages into cleanup', async () => {
const { run } = await admit();
deps.resolvePrincipal = async () => {
throw new Error('permissions unavailable');
};
await runtime.initialize();
expect(retired).toHaveLength(0);
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
state: 'queued',
cancellationRequested: false,
leaseEpoch: 0,
});
expect(deps.onError).toHaveBeenCalledTimes(1);
});
it('enumerates only bounded metadata in system context then scopes every full read to its tenant', async () => {
await admit('one');
await admit('two');
await admit('three');
const metadata = await runAsSystem(() => storage.listPendingMetadata(undefined, 2));
expect(metadata.runs).toHaveLength(2);
expect(metadata.next).toBeDefined();
expect(Object.keys(metadata.runs[0]).sort()).toEqual(['channelId', 'runId', 'tenantId']);
const read = jest.spyOn(storage, 'read');
allowed = false;
await runtime.initialize();
expect(retired).toHaveLength(2);
expect(read.mock.calls.every(([tenant]) => tenant === tenantId)).toBe(true);
expect(await scope(() => db.getAgentChannelMetadata('channel'))).not.toHaveProperty(
'encryptedBotToken',
);
});
it('waits for owned cleanup on shutdown and does not start the next run', async () => {
await admit('one');
await admit('two');
allowed = false;
let entered!: () => void;
let release!: () => void;
const ready = new Promise<void>((resolve) => {
entered = resolve;
});
const gate = new Promise<void>((resolve) => {
release = resolve;
});
deps.retireForAuthorityLoss = async (run) => {
retired.push(run);
entered();
await gate;
return 'stopped';
};
const initializing = runtime.initialize();
await ready;
let ended = false;
const stopping = runtime.stop().then(() => {
ended = true;
});
await new Promise<void>((resolve) => setImmediate(resolve));
expect(ended).toBe(false);
release();
await initializing;
await stopping;
expect(retired).toHaveLength(1);
});
it('uses an allowlist so future top-level or Slack credential fields cannot enter authority metadata', async () => {
await mongoose.models.AgentChannel.collection.updateOne(
{ tenantId, channelId: 'channel' },
{ $set: { futureCredential: 'must-not-leak', 'slack.futureCredential': 'must-not-leak' } },
);
const metadata = await scope(() => db.getAgentChannelMetadata('channel'));
expect(Object.keys(metadata!).sort()).toEqual([
'agentId',
'channelId',
'createdBy',
'credentialMode',
'revision',
'slack',
'status',
'tenantId',
'type',
]);
expect(metadata?.slack).toEqual({ teamId: 'T1', appId: 'A1', botUserId: 'UBOT' });
expect(JSON.stringify(metadata)).not.toContain('must-not-leak');
});
it('defines bounded cleanup configuration without exposing the algorithmic claim batch as a policy knob', () => {
const config = channelConfigSchema.parse({});
expect(config.cleanup).toEqual({ batchSize: 25, intervalMs: 1000, maxIdleIntervalMs: 30_000 });
expect(config.claimBatchSize).toBe(25);
for (const cleanup of [
{ batchSize: 0 },
{ batchSize: 101 },
{ intervalMs: 999 },
{ maxIdleIntervalMs: 300_001 },
]) {
expect(channelConfigSchema.safeParse({ cleanup }).success).toBe(false);
}
});
it('keeps tenantless work authorized and deletes its closed parent in the application default tenant', async () => {
const defaultScope = <T>(work: () => Promise<T>) =>
tenantStorage.run({ tenantId: undefined }, work);
await defaultScope(async () => {
await db.createAgentChannel({
channelId: 'default-channel',
agentId: 'agent',
createdBy: 'user-a',
type: 'slack',
credentialMode: 'per_user',
status: 'active',
revision: 'v1',
slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' },
});
await mongoose.models.ChannelIdentity.create({
tenantId: '_',
linkId: 'link',
revision: 'link-v1',
state: 'active',
userId: 'user-a',
ticketHash: 'default-ticket',
ticketExpiresAtMs: clock + 60000,
expiresAtMs: clock + 600000,
identity: {
tenantId: '_',
channelId: 'default-channel',
channelRevision: 'v1',
teamId: 'T1',
externalUserId: 'U1',
agentId: 'agent',
},
});
await storage.admit(
{
tenantId: '_',
channelId: 'default-channel',
agentId: 'agent',
principal: {
userId: 'user-a',
channelRevision: 'v1',
linkId: 'link',
linkRevision: 'link-v1',
},
admission: {
eventId: 'default-event',
externalChannelId: 'C1',
externalThreadKey: 'C1:1',
externalMessageKey: 'C1:1',
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
input: 'default tenant input',
},
},
clock,
);
});
await runtime.initialize();
expect(retired).toHaveLength(0);
expect(deps.onError).not.toHaveBeenCalled();
await defaultScope(() => db.beginAgentChannelDeletion('default-channel', 'user-a'));
await restart();
expect(retired).toHaveLength(1);
expect(retired[0].tenantId).toBe('_');
expect(await defaultScope(() => db.getAgentChannelMetadata('default-channel'))).toBeNull();
expect(deps.onError).not.toHaveBeenCalled();
});
it('caps maintenance pages by the storage policy and never turns a credential lookup outage into retirement', async () => {
await admit('one');
await admit('two');
deps.getConfig = async () =>
channelConfigSchema.parse({
enabled: true,
maxMetadataPageSize: 1,
cleanup: { batchSize: 2, intervalMs: 60000, maxIdleIntervalMs: 60000 },
});
const read = jest.spyOn(storage, 'listPendingMetadata');
deps.isCredentialCurrent = async () => {
throw new Error('credential store unavailable');
};
await runtime.initialize();
expect(read).toHaveBeenCalledWith(undefined, 1);
expect(retired).toHaveLength(0);
expect(deps.onError).toHaveBeenCalledTimes(1);
});
it('holds an invalid issued credential lane until exact retirement confirms provider drain', async () => {
const { run } = await admit('expired-credential');
const claimed = (await scope(() => storage.claim(tenantId, 'channel', 'gateway', clock, 60000)))!;
const fence = { runId: run.runId, owner: 'gateway', epoch: claimed.leaseEpoch };
await scope(() => storage.prepareExecution(tenantId, fence, clock));
await scope(() =>
storage.attachResponseCredential(
tenantId,
fence,
{ apiKeyId: 'expired-key', encryptedApiKey: 'encrypted-original' },
clock,
),
);
deps.isCredentialCurrent = async (current) => {
expect(current.execution?.apiKeyId).toBe('expired-key');
return false;
};
stoppedResult = 'pending';
await runtime.initialize();
expect(retired).toHaveLength(1);
expect(retired[0].execution?.apiKeyId).toBe('expired-key');
expect(
await scope(() => mongoose.models.ChannelLane.findOne({ activeRunId: run.runId }).lean()),
).not.toBeNull();
expect((await scope(() => storage.read(tenantId, 'channel', run.runId)))?.state).not.toBe(
'cancelled',
);
clock += 60001;
stoppedResult = 'stopped';
await restart();
expect(retired).toHaveLength(2);
expect((await scope(() => storage.read(tenantId, 'channel', run.runId)))?.state).toBe(
'cancelled',
);
expect(
await scope(() => mongoose.models.ChannelLane.findOne({ activeRunId: run.runId }).lean()),
).toBeNull();
expect(deps.onError).not.toHaveBeenCalled();
});
it.each(['read', 'parse'] as const)(
'retries failed %s initialization with one shared attempt',
async (failure) => {
await admit();
allowed = false;
const original = deps.getConfig;
let fail = true;
const read = jest.fn(async () => {
if (fail) {
if (failure === 'read') throw new Error('configuration unavailable');
const invalid = channelConfigSchema.parse({});
invalid.cleanup.intervalMs = 0;
return invalid;
}
return original();
});
deps.getConfig = read;
const first = runtime.initialize();
expect(runtime.initialize()).toBe(first);
await expect(first).rejects.toThrow();
fail = false;
const retry = runtime.initialize();
expect(runtime.initialize()).toBe(retry);
await retry;
expect(retired).toHaveLength(1);
const calls = read.mock.calls.length;
await runtime.initialize();
expect(read).toHaveBeenCalledTimes(calls);
},
);
it('keeps stop permanent after an initialization failure', async () => {
const read = jest.fn(async () => {
throw new Error('configuration unavailable');
});
deps.getConfig = read;
await expect(runtime.initialize()).rejects.toThrow('configuration unavailable');
await runtime.stop();
await runtime.initialize();
runtime.wake();
expect(read).toHaveBeenCalledTimes(1);
expect(retired).toHaveLength(0);
});
it('disposes a partially started worker before retrying initialization', async () => {
await admit();
allowed = false;
const create = recovery.createIdleRecoveryLoop;
const partial = create({
intervalMs: 60000,
maxIdleIntervalMs: 60000,
scan: async () => true,
onError: jest.fn(),
});
const start = jest
.spyOn(partial, 'start')
.mockRejectedValueOnce(new Error('initial scan failed'));
const stop = jest.spyOn(partial, 'stop');
const factory = jest.spyOn(recovery, 'createIdleRecoveryLoop').mockReturnValueOnce(partial);
try {
await expect(runtime.initialize()).rejects.toThrow('initial scan failed');
expect(start).toHaveBeenCalledTimes(1);
expect(stop).toHaveBeenCalledTimes(1);
await runtime.initialize();
expect(retired).toHaveLength(1);
expect(factory).toHaveBeenCalledTimes(2);
} finally {
factory.mockRestore();
start.mockRestore();
stop.mockRestore();
}
});
it('does not reopen when a pending initialization rejects during shutdown', async () => {
let reject!: (error: Error) => void;
const gate = new Promise<Awaited<ReturnType<typeof deps.getConfig>>>((_resolve, fail) => {
reject = fail;
});
const read = jest.fn(() => gate);
deps.getConfig = read;
const initializing = runtime.initialize();
const stopping = runtime.stop();
reject(new Error('startup unavailable'));
await expect(initializing).rejects.toThrow('startup unavailable');
await stopping;
await runtime.initialize();
expect(read).toHaveBeenCalledTimes(1);
expect(retired).toHaveLength(0);
});
it.each([false, true])(
'releases an expired Response lane only after canonical retirement proof: pending=%s',
async (pending) => {
await admit();
const claimed = (await scope(() =>
storage.claim(tenantId, 'channel', 'worker', clock, 60000),
))!;
const fence = { runId: claimed.runId, owner: claimed.leaseOwner!, epoch: claimed.leaseEpoch };
const prepared = await scope(() => storage.prepareExecution(tenantId, fence, clock));
await scope(() =>
storage.attachResponseCredential(
tenantId,
fence,
{ apiKeyId: 'issued', encryptedApiKey: 'encrypted' },
clock,
),
);
const responses = createResponseStorage(mongoose);
const execution = prepared.execution!;
const owner = { userId: prepared.principal.userId, tenantId };
const responseId = `resp_${prepared.runId}`;
const reservation = await scope(() =>
responses.reserve({
...owner,
responseId,
requestKey: `channel:${prepared.runId}`,
sourceKeyId: 'issued',
fingerprint: 'cleanup',
agentId: prepared.agentId,
requestJSON: JSON.stringify({
model: prepared.agentId,
input: prepared.admission.input,
background: true,
store: true,
...(execution.parentMessageId === '00000000-0000-0000-0000-000000000000'
? {}
: { previous_response_id: execution.parentMessageId }),
}),
conversationId: execution.conversationId,
parentMessageId: execution.parentMessageId,
}),
);
if (!reservation.ok) throw new Error('Reservation failed');
await scope(() => storage.bindResponse(tenantId, fence, responseId, clock));
await scope(() =>
mongoose.models.StoredResponse.updateOne(
{ responseId },
{ $set: { expiredAt: new Date(0), ...(pending ? { generationCreatedAt: 1234 } : {}) } },
),
);
const jobs = {
getJob: jest.fn(async () => undefined),
abortJob: jest.fn(),
finishProviderTerminalPersistence: jest.fn(),
};
const adapter = createChannelExecutionAdapter({
responseProjection: async () => {
throw new Error('Cleanup must not project or execute');
},
responseRetirement: createChannelResponseRetirement({
storage: responses,
recovery: storage,
retireResponse: createResponseRetirement({ storage: responses, jobs }),
}),
});
deps.retireForAuthorityLoss = adapter.retireForAuthorityLoss;
deps.isCredentialCurrent = async () => false;
await runtime.initialize();
await runtime.stop();
const next = await admit('next');
const nextClaim = await scope(() =>
storage.claim(tenantId, 'channel', 'next-worker', clock + 60001, 60000),
);
if (pending) {
expect(nextClaim?.runId).toBe(claimed.runId);
expect(nextClaim?.errorCode).toBe('channel_authority_changed');
} else expect(nextClaim?.runId).toBe(next.run.runId);
expect(jobs.abortJob).not.toHaveBeenCalled();
},
);

View file

@ -0,0 +1,155 @@
import { randomUUID } from 'node:crypto';
import { channelConfigSchema } from 'librechat-data-provider';
import { runAsSystem, tenantStorage } from '@librechat/data-schemas';
import type { AgentChannelMethods, ChannelRunStorage } from '@librechat/data-schemas';
import type { ChannelConfig } from 'librechat-data-provider';
import type { ChannelRunDependencies } from './runs';
import { createIdleRecoveryLoop } from '~/agents/recovery';
import { hasChannelRunAuthority } from './runs';
export interface ChannelCleanupDependencies {
storage: ChannelRunStorage;
channels: Pick<
AgentChannelMethods,
'getAgentChannelMetadata' | 'listDeletingAgentChannels' | 'deleteAgentChannel'
>;
resolvePrincipal: ChannelRunDependencies['resolvePrincipal'];
retireForAuthorityLoss: ChannelRunDependencies['retireForAuthorityLoss'];
isCredentialCurrent: (
run: Parameters<ChannelRunDependencies['retireForAuthorityLoss']>[0],
) => Promise<boolean>;
deleteApiKey: (id: string, userId: string) => Promise<boolean>;
getConfig: () => Promise<ChannelConfig | undefined>;
onError: (error: unknown) => void;
now?: () => number;
}
/** Private recovery survives disabled features and missing gateway/channel rows.
* Only metadata crosses system context; cleanup cannot enqueue or deliver. */
export function createChannelCleanupRuntime(deps: ChannelCleanupDependencies) {
const owner = `channel-cleanup-${randomUUID()}`;
const now = deps.now ?? Date.now;
let worker: ReturnType<typeof createIdleRecoveryLoop> | undefined;
let initialization: Promise<void> | undefined;
let stopped = false;
let runCursor: string | undefined;
let channelCursor: string | undefined;
return {
initialize(): Promise<void> {
if (stopped) return Promise.resolve();
if (initialization) return initialization;
initialization = (async () => {
const initial = channelConfigSchema.parse((await deps.getConfig()) ?? {});
if (stopped) return;
worker = createIdleRecoveryLoop({
...initial.cleanup,
onError: deps.onError,
scan: async () => {
if (stopped) return true;
const config = channelConfigSchema.parse((await deps.getConfig()) ?? {});
const pageSize = Math.min(config.cleanup.batchSize, config.maxMetadataPageSize);
const [runs, deleting] = await runAsSystem(() =>
Promise.all([
deps.storage.listPendingMetadata(runCursor, pageSize),
deps.channels.listDeletingAgentChannels(channelCursor, pageSize),
]),
);
runCursor = runs.next;
channelCursor = deleting.next;
let idle = !runs.next && !deleting.next;
for (const target of runs.runs) {
if (stopped) break;
await tenantStorage.run(
{ tenantId: target.tenantId === '_' ? undefined : target.tenantId },
async () => {
try {
const run = await deps.storage.read(
target.tenantId,
target.channelId,
target.runId,
);
if (!run) return;
const channel = await deps.channels.getAgentChannelMetadata(target.channelId);
if (
config.enabled &&
channel &&
(await hasChannelRunAuthority(channel, run, deps.resolvePrincipal)) &&
(await deps.isCredentialCurrent(run))
)
return;
const claimed = await deps.storage.claimAuthorityLost(
run,
owner,
now(),
config.leaseMs,
);
if (!claimed) return;
idle = false;
if ((await deps.retireForAuthorityLoss(claimed)) !== 'stopped') return;
await deps.storage.finishAuthorityLost(
claimed.tenantId,
{
runId: claimed.runId,
owner,
epoch: claimed.leaseEpoch,
},
now(),
);
} catch (error) {
idle = false;
deps.onError(error);
}
},
);
}
for (const target of deleting.channels) {
if (stopped) break;
await tenantStorage.run({ tenantId: target.tenantId || undefined }, async () => {
try {
const channel = await deps.channels.getAgentChannelMetadata(target.channelId);
if (channel?.status !== 'deleting') return;
if (channel.apiKeyId)
await deps.deleteApiKey(channel.apiKeyId, channel.createdBy);
const result = await deps.channels.deleteAgentChannel(
channel.channelId,
channel.createdBy,
);
if (result.status !== 'not_found') idle = false;
} catch (error) {
idle = false;
deps.onError(error);
}
});
}
return idle;
},
});
await worker.start();
if (stopped) await worker.stop();
})().catch(async (error: unknown) => {
const failed = worker;
worker = undefined;
try {
await failed?.stop();
} catch {
/* Preserve the initialization failure after attempting owned worker disposal. */
}
initialization = undefined;
throw error;
});
return initialization;
},
wake(): void {
worker?.wake();
},
async stop(): Promise<void> {
stopped = true;
try {
await initialization;
} catch {
/* Failed initialization has no running worker. */
}
await worker?.stop();
},
};
}

View file

@ -0,0 +1,209 @@
import { z } from 'zod';
import { Constants } from 'librechat-data-provider';
import type {
AgentApiKeyBinding,
AgentApiKeyListItem,
AgentChannelRecord,
ChannelRunRecord,
ChannelRunStorage,
ResponseOwner,
ResponseStorage,
IUser,
} from '@librechat/data-schemas';
import type { Request } from 'express';
import type { ChannelRunDependencies } from './runs';
import { hasChannelRunAuthority } from './runs';
export interface ChannelResponseCredentialDependencies {
encrypt: (value: string) => Promise<string>;
decrypt: (value: string) => Promise<string>;
storage: Pick<ChannelRunStorage, 'read' | 'attachResponseCredential'>;
responses: Pick<ResponseStorage, 'read'>;
getChannel: (id: string) => Promise<AgentChannelRecord | null>;
resolvePrincipal: ChannelRunDependencies['resolvePrincipal'];
createKey: (input: {
userId: string;
name: string;
expiresAt: Date;
binding: AgentApiKeyBinding;
}) => Promise<{ id: string; key: string }>;
getKey: (keyId: string, userId: string) => Promise<AgentApiKeyListItem | null>;
deleteKey: (keyId: string, userId: string) => Promise<boolean>;
lifetimeMs: () => Promise<number>;
now?: () => number;
}
const requestSchema = z
.object({
model: z.string(),
input: z.string(),
background: z.literal(true),
store: z.literal(true),
stream: z.literal(false).optional(),
previous_response_id: z.string().optional(),
})
.strict();
const bindingFor = (run: ChannelRunRecord): AgentApiKeyBinding => ({
kind: 'channel_run',
id: `${run.channelId}/${run.runId}`,
});
const previousResponse = (run: ChannelRunRecord) =>
run.execution?.parentMessageId === Constants.NO_PARENT
? undefined
: run.execution?.parentMessageId;
/** Raw keys cross only the guarded gateway boundary; durable state retains encrypted keys and immutable grant references. */
export function createChannelResponseCredentials(deps: ChannelResponseCredentialDependencies) {
const now = deps.now ?? Date.now;
async function current(
binding: AgentApiKeyBinding,
keyId: string,
owner: ResponseOwner,
agentId?: string,
) {
if (binding.kind !== 'channel_run' || !keyId) return null;
const [channelId, runId, extra] = binding.id.split('/');
if (!channelId || !runId || extra != null) return null;
const run = await deps.storage.read(owner.tenantId || '_', channelId, runId);
if (
!run ||
run.principal.userId !== owner.userId ||
run.execution?.apiKeyId !== keyId ||
run.execution.protocol !== 'responses' ||
(agentId != null && run.agentId !== agentId)
)
return null;
const [authorized, key] = await Promise.all([
deps
.getChannel(channelId)
.then((channel) =>
channel ? hasChannelRunAuthority(channel, run, deps.resolvePrincipal) : false,
),
deps.getKey(keyId, run.principal.userId),
]);
// A key read early must still be unexpired after the independent policy lookup finishes.
return authorized && eligibleKey(run, key) ? run : null;
}
function eligibleKey(run: ChannelRunRecord, key: AgentApiKeyListItem | null): boolean {
const execution = run.execution;
if (!execution?.apiKeyId || execution.protocol !== 'responses' || !execution.encryptedApiKey)
return false;
const binding = bindingFor(run);
return Boolean(
key &&
key.id === execution.apiKeyId &&
key.binding?.kind === binding.kind &&
key.binding.id === binding.id &&
(key.expiresAt == null ||
(Number.isFinite(new Date(key.expiresAt).getTime()) &&
new Date(key.expiresAt).getTime() > now())),
);
}
async function isCurrent(run: ChannelRunRecord): Promise<boolean> {
const execution = run.execution;
if (!execution?.apiKeyId) return true;
if (execution.protocol !== 'responses' || !execution.encryptedApiKey) return false;
return eligibleKey(run, await deps.getKey(execution.apiKeyId, run.principal.userId));
}
async function reveal(run: ChannelRunRecord): Promise<string> {
const execution = run.execution;
if (!execution?.apiKeyId || !execution.encryptedApiKey)
throw new Error('Response credential unavailable');
if (!(await isCurrent(run))) throw new Error('Response credential authority changed');
return deps.decrypt(execution.encryptedApiKey);
}
return {
isCurrent: (run: ChannelRunRecord): Promise<boolean> => isCurrent(run),
async provision(run: ChannelRunRecord): Promise<string> {
if (run.execution?.apiKeyId) return reveal(run);
if (
!run.execution ||
run.execution.deliveryKey ||
run.execution.enqueueAttemptedAt ||
run.cancellationRequested ||
!run.leaseOwner
)
throw new Error('Response credential admission changed');
const lifetimeMs = await deps.lifetimeMs();
if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs <= 0)
throw new Error('Invalid response credential policy');
const key = await deps.createKey({
userId: run.principal.userId,
name: 'Channel response run',
expiresAt: new Date(now() + lifetimeMs),
binding: bindingFor(run),
});
let saved: ChannelRunRecord;
try {
saved = await deps.storage.attachResponseCredential(
run.tenantId,
{ runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch },
{ apiKeyId: key.id, encryptedApiKey: await deps.encrypt(key.key) },
now(),
);
} catch (error) {
const currentRun = await deps.storage.read(run.tenantId, run.channelId, run.runId);
const winner = currentRun?.execution;
const sameAdmission =
currentRun &&
winner?.apiKeyId &&
winner.encryptedApiKey &&
winner.protocol === 'responses' &&
currentRun.principal.userId === run.principal.userId &&
currentRun.agentId === run.agentId &&
winner.requestId === run.execution.requestId &&
currentRun.leaseOwner === run.leaseOwner &&
currentRun.leaseEpoch === run.leaseEpoch;
if (!sameAdmission) {
if (winner?.apiKeyId !== key.id) await deps.deleteKey(key.id, run.principal.userId);
throw error;
}
saved = currentRun;
}
if (saved.execution?.apiKeyId !== key.id) await deps.deleteKey(key.id, run.principal.userId);
return reveal(saved);
},
async authorizeSource(
binding: AgentApiKeyBinding,
keyId: string,
owner: ResponseOwner,
agentId: string,
): Promise<boolean> {
return Boolean(await current(binding, keyId, owner, agentId));
},
async authorizeRequest(input: {
keyId: string;
binding: AgentApiKeyBinding;
user: IUser;
request: Request;
}): Promise<boolean> {
const owner = { userId: String(input.user._id), tenantId: input.user.tenantId };
const run = await current(input.binding, input.keyId, owner);
if (!run) return false;
const pathname = input.request.originalUrl.split('?')[0];
if (input.request.method === 'POST' && pathname === '/api/agents/v1/responses') {
if (run.cancellationRequested) return false;
const parsed = requestSchema.safeParse(input.request.body);
return (
parsed.success &&
parsed.data.model === run.agentId &&
parsed.data.input === run.admission.input &&
parsed.data.previous_response_id === previousResponse(run) &&
input.request.headers['idempotency-key'] === `channel:${run.runId}`
);
}
const path = /^\/api\/agents\/v1\/responses\/(resp_[\w-]+)(\/cancel)?$/.exec(pathname);
if (!path || (path[2] ? input.request.method !== 'POST' : input.request.method !== 'GET'))
return false;
if (path[2] && !run.cancellationRequested) return false;
const response = await deps.responses.read(owner, path[1]);
return Boolean(
response &&
response.agentId === run.agentId &&
response.requestKey === `channel:${run.runId}` &&
response.sourceKeyId === input.keyId &&
(run.execution?.responseId == null || run.execution.responseId === response.responseId),
);
},
};
}

View file

@ -0,0 +1,19 @@
import { ChannelExecutionError as CompatibilityError } from './execution';
import { ChannelExecutionError } from './errors';
it.each([
'execution_not_prepared',
'execution_identity_changed',
'dispatch_outcome_unknown',
'generation_missing',
'generation_replaced',
'stop_unconfirmed',
] as const)('preserves error identity and diagnostics for %s', (code) => {
expect(CompatibilityError).toBe(ChannelExecutionError);
const error = new ChannelExecutionError(code);
expect(error).toBeInstanceOf(Error);
expect(error).toBeInstanceOf(CompatibilityError);
expect(error.name).toBe('ChannelExecutionError');
expect(error.message).toBe(code);
expect(error.code).toBe(code);
});

View file

@ -0,0 +1,14 @@
export class ChannelExecutionError extends Error {
constructor(
readonly code:
| 'execution_not_prepared'
| 'execution_identity_changed'
| 'dispatch_outcome_unknown'
| 'generation_missing'
| 'generation_replaced'
| 'stop_unconfirmed',
) {
super(code);
this.name = 'ChannelExecutionError';
}
}

View file

@ -0,0 +1,256 @@
import express from 'express';
import request from 'supertest';
import mongoose from 'mongoose';
import { Constants } from 'librechat-data-provider';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createModels,
createMethods,
createChannelRunStorage,
tenantStorage,
} from '@librechat/data-schemas';
import type { ChannelRunRecord } from '@librechat/data-schemas';
import type { ChannelExecutionDependencies } from './execution';
import { createChannelExecutionAdapter } from './execution';
import { createChannelHandlers } from './service';
import { createChannelRunHandlers } from './runs';
const scope = <T>(work: () => Promise<T>) => tenantStorage.run({ tenantId: 'tenant-a' }, work);
const makeRun = (): ChannelRunRecord => ({
tenantId: 'tenant-a',
channelId: 'channel-a',
agentId: 'agent-a',
runId: 'channel-run-a',
laneId: 'lane-a',
state: 'executing',
sequence: 1,
receivedAt: Date.now(),
payloadHash: 'hash-a',
principal: {
userId: 'user-a',
channelRevision: 'revision-a',
linkId: 'link-a',
linkRevision: 'link-rev-a',
},
admission: {
eventId: 'event-a',
externalChannelId: 'C1',
externalThreadKey: 'C1:1.001',
externalMessageKey: 'C1:1.001',
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
input: 'hello',
},
cancellationRequested: false,
receipts: [],
leaseEpoch: 1,
leaseExpiresAt: Date.now() + 60_000,
execution: {
requestId: 'channel-run-a',
conversationId: 'conversation-a',
parentMessageId: String(Constants.NO_PARENT),
},
});
describe('mounted service-token advance cannot execute an unbound run', () => {
let mongo: MongoMemoryReplSet;
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } });
await mongoose.connect(mongo.getUri());
createModels(mongoose);
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
it('rejects legacy and unprepared state behind the actual service guard without starting work', () =>
scope(async () => {
const fixture = makeRun();
await createMethods(mongoose).createAgentChannel({
channelId: fixture.channelId,
tenantId: fixture.tenantId,
agentId: fixture.agentId,
createdBy: fixture.principal.userId,
type: 'slack',
status: 'active',
credentialMode: 'per_user',
revision: fixture.principal.channelRevision,
slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' },
});
await mongoose.models.ChannelIdentity.create({
tenantId: fixture.tenantId,
linkId: fixture.principal.linkId,
revision: fixture.principal.linkRevision,
userId: fixture.principal.userId,
state: 'active',
ticketHash: 'ticket',
ticketExpiresAtMs: Date.now() + 60000,
expiresAtMs: Date.now() + 60000,
identity: {
tenantId: fixture.tenantId,
channelId: fixture.channelId,
channelRevision: fixture.principal.channelRevision,
teamId: 'T1',
externalUserId: 'U1',
agentId: fixture.agentId,
},
});
const storage = createChannelRunStorage(mongoose);
await storage.admit(
{
tenantId: fixture.tenantId,
channelId: fixture.channelId,
agentId: fixture.agentId,
principal: fixture.principal,
admission: fixture.admission,
},
Date.now(),
);
const run = (await storage.claim(
fixture.tenantId,
fixture.channelId,
'worker',
Date.now(),
60_000,
))!;
const token = 'test-service-credential-at-least-32-characters';
const handlers = createChannelHandlers({
db: createMethods(mongoose),
serviceToken: () => token,
config: async () => ({ enabled: true }),
getAgent: async () => ({ id: fixture.agentId }),
deleteApiKey: async () => true,
verifySlackToken: async () => null,
logError: jest.fn(),
});
const responseProjection = jest.fn();
const adapter = createChannelExecutionAdapter({
responseRetirement: jest.fn(),
responseProjection,
});
const runs = createChannelRunHandlers({
storage,
guard: handlers.service,
resolvePrincipal: async () => fixture.principal,
advance: adapter.advance,
retireForAuthorityLoss: adapter.retireForAuthorityLoss,
});
const app = express();
app.use(express.json());
app.post('/service/:channelId/runs/:runId/advance', runs.advance);
const route = `/service/${run.channelId}/runs/${run.runId}/advance`;
const body = { fence: { runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch } };
await request(app).post(route).send(body).expect(401);
for (let attempt = 0; attempt < 2; attempt++) {
await request(app)
.post(route)
.auth(token, { type: 'bearer' })
.set('X-Tenant-Id', fixture.tenantId)
.send(body)
.expect(409, { error: 'execution_not_prepared' });
}
const held = await storage.read(run.tenantId, run.channelId, run.runId);
expect(held?.execution?.enqueueAttemptedAt).toBeUndefined();
expect(held?.execution?.deliveryKey).toBeUndefined();
expect(held?.snapshot).toBeUndefined();
expect(responseProjection).not.toHaveBeenCalled();
}));
});
describe('Responses-only Channels execution', () => {
let run: ChannelRunRecord;
let deps: ChannelExecutionDependencies;
let adapter: ReturnType<typeof createChannelExecutionAdapter>;
beforeEach(() => {
run = makeRun();
deps = {
responseProjection: jest.fn(async () => ({
snapshot: { status: 'running' as const, text: '', elapsedMs: 1 },
})),
responseRetirement: jest.fn(async () => 'pending' as const),
};
adapter = createChannelExecutionAdapter(deps);
});
function loseAuthority() {
run.cancellationRequested = true;
run.errorCode = 'channel_authority_changed';
}
it.each(['unprepared', 'unbound', 'legacy_dispatched', 'responses_unbound'] as const)(
'refuses %s advance without projecting a Response',
async (kind) => {
if (kind === 'unprepared') run.execution = undefined;
if (kind === 'legacy_dispatched') run.execution!.deliveryKey = 'old-trigger';
if (kind === 'responses_unbound') run.execution!.protocol = 'responses';
await expect(adapter.advance(run)).rejects.toMatchObject({ code: 'execution_not_prepared' });
expect(deps.responseProjection).not.toHaveBeenCalled();
},
);
it('delegates a bound Response to its canonical projection and retirement', async () => {
run.execution = { ...run.execution!, protocol: 'responses', responseId: 'resp_saved' };
await expect(adapter.advance(run)).resolves.toMatchObject({ snapshot: { status: 'running' } });
expect(deps.responseProjection).toHaveBeenCalledWith(run);
loseAuthority();
await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('pending');
expect(deps.responseRetirement).toHaveBeenCalledWith(run);
});
it.each(['not_cancelled', 'different_reason'] as const)(
'rejects %s cleanup before consulting Responses',
async (kind) => {
loseAuthority();
if (kind === 'not_cancelled') run.cancellationRequested = false;
if (kind === 'different_reason') run.errorCode = 'other';
await expect(adapter.retireForAuthorityLoss(run)).rejects.toMatchObject({
code: 'execution_identity_changed',
});
expect(deps.responseRetirement).not.toHaveBeenCalled();
},
);
it.each(['unprepared', 'unprovisioned', 'responses_unprovisioned'] as const)(
'retires %s authority loss without a possible provider admission',
async (kind) => {
loseAuthority();
if (kind === 'unprepared') run.execution = undefined;
if (kind === 'responses_unprovisioned') run.execution!.protocol = 'responses';
await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('stopped');
expect(deps.responseRetirement).not.toHaveBeenCalled();
},
);
it('reconciles an issued credential even when Response binding has not finished', async () => {
loseAuthority();
run.execution = { ...run.execution!, protocol: 'responses', apiKeyId: 'issued' };
await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('pending');
expect(deps.responseRetirement).toHaveBeenCalledWith(run);
});
it.each([
'deliveryKey',
'enqueueAttemptedAt',
'generationCreatedAt',
'apiKeyId',
'responseId',
] as const)(
'holds unknown execution with %s instead of guessing that it stopped',
async (field) => {
loseAuthority();
Object.assign(run.execution!, { [field]: field.endsWith('At') ? 100 : 'unknown' });
await expect(adapter.retireForAuthorityLoss(run)).rejects.toMatchObject({
code: 'execution_identity_changed',
});
expect(deps.responseRetirement).not.toHaveBeenCalled();
},
);
it('preserves canonical retirement errors for recovery to retry', async () => {
loseAuthority();
run.execution = { ...run.execution!, protocol: 'responses', responseId: 'resp_saved' };
const error = new Error('storage_unavailable');
jest.mocked(deps.responseRetirement).mockRejectedValue(error);
await expect(adapter.retireForAuthorityLoss(run)).rejects.toBe(error);
});
});

View file

@ -0,0 +1,43 @@
import type { ChannelRunSnapshot } from 'librechat-data-provider';
import type { ChannelRunRecord } from '@librechat/data-schemas';
import type { createChannelResponseProjection, createChannelResponseRetirement } from './responses';
import { ChannelExecutionError } from './errors';
export { ChannelExecutionError } from './errors';
export interface ChannelExecutionDependencies {
responseProjection: ReturnType<typeof createChannelResponseProjection>;
responseRetirement: ReturnType<typeof createChannelResponseRetirement>;
}
/** Execution and cancellation belong to Responses; Channels only projects delivery state. */
export function createChannelExecutionAdapter(deps: ChannelExecutionDependencies) {
return {
async retireForAuthorityLoss(run: ChannelRunRecord): Promise<'stopped' | 'pending'> {
if (!run.cancellationRequested || run.errorCode !== 'channel_authority_changed')
throw new ChannelExecutionError('execution_identity_changed');
if (!run.execution) return 'stopped';
const execution = run.execution;
// A credential permits Response admission even before its ID has been bound.
// Only an unprovisioned run can be retired without consulting Responses.
if (
!execution.apiKeyId &&
!execution.responseId &&
execution.generationCreatedAt == null &&
execution.deliveryKey == null &&
execution.enqueueAttemptedAt == null
)
return 'stopped';
if (execution.protocol !== 'responses')
throw new ChannelExecutionError('execution_identity_changed');
return deps.responseRetirement(run);
},
async advance(
run: ChannelRunRecord,
): Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }> {
if (run.execution?.protocol !== 'responses' || !run.execution.responseId)
throw new ChannelExecutionError('execution_not_prepared');
return deps.responseProjection(run);
},
};
}

View file

@ -0,0 +1,389 @@
import mongoose from 'mongoose';
import { MongoMemoryReplSet } from 'mongodb-memory-server';
import {
createModels,
createMethods,
createResponseStorage,
tenantStorage,
runAsSystem,
} from '@librechat/data-schemas';
import type {
ChannelRunRecord,
ResponseStorage,
StoredResponseRecord,
} from '@librechat/data-schemas';
import type { Response } from '~/agents/responses/types';
import {
createResponseCompletion,
createResponseExecutionConfirmation,
createResponseExecutionRelease,
} from '~/agents/responses/completion';
import {
buildPendingBackgroundResponse,
createBackgroundResponseService,
} from '~/agents/responses/background';
import { createChannelResponseProjection, createChannelResponseRetirement } from './responses';
import { createResponseRetirement } from '~/agents/responses/retirement';
import { GenerationJobManager } from '~/stream';
let mongo: MongoMemoryReplSet;
let storage: ResponseStorage;
let stored: StoredResponseRecord;
let run: ChannelRunRecord;
const owner = { userId: 'user', tenantId: 'projection-test' };
const scoped = <T>(work: () => Promise<T>) => tenantStorage.run(owner, work);
beforeAll(async () => {
mongo = await MongoMemoryReplSet.create({
replSet: { count: 1 },
instanceOpts: [{ args: ['--nounixsocket'] }],
});
await mongoose.connect(mongo.getUri(), { autoIndex: false });
createModels(mongoose);
storage = createResponseStorage(mongoose);
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
});
beforeEach(async () => {
await runAsSystem(async () => {
await mongoose.models.StoredResponse.deleteMany({});
await mongoose.models.Conversation.deleteMany({});
await mongoose.models.ResponseScope?.deleteMany({});
});
const reserved = await scoped(() =>
storage.reserve({
...owner,
responseId: 'resp_saved',
requestKey: 'channel:run',
fingerprint: 'request',
agentId: 'agent',
sourceKeyId: 'key',
requestJSON: JSON.stringify({
model: 'agent',
input: 'hello',
background: true,
store: true,
}),
conversationId: 'conversation',
parentMessageId: 'root',
}),
);
if (!reserved.ok) throw new Error('Reservation failed');
stored = reserved.record;
run = {
tenantId: owner.tenantId,
channelId: 'channel',
agentId: 'agent',
runId: 'run',
laneId: 'lane',
state: 'executing',
sequence: 1,
payloadHash: 'payload',
receivedAt: 1000,
principal: { userId: owner.userId, channelRevision: 'revision' },
admission: {
eventId: 'event',
externalChannelId: 'C1',
externalThreadKey: 'C1:1.0',
externalMessageKey: 'C1:1.1',
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
input: 'hello',
},
cancellationRequested: false,
receipts: [],
leaseEpoch: 1,
leaseExpiresAt: Date.now() + 10000,
execution: {
protocol: 'responses',
responseId: stored.responseId,
requestId: 'run',
conversationId: stored.conversationId,
parentMessageId: stored.parentMessageId,
},
};
});
function fixture() {
let authorized = true;
const cancel = jest.fn(async () => {});
const service = createBackgroundResponseService({
storage,
recoverTerminal: createResponseCompletion({
storage,
cleanupCheckpoint: async () => {},
confirmExecution: createResponseExecutionConfirmation(GenerationJobManager),
releaseExecution: createResponseExecutionRelease(GenerationJobManager),
}).finish,
authorize: async () => authorized,
enqueue: async () => {
throw new Error('Projection must not enqueue');
},
resolveContinuation: async () => {
throw new Error('Projection must not rebuild history');
},
observe: async () => null,
cancel,
});
return {
project: createChannelResponseProjection({
storage,
readResponse: service.retrieve,
cancelResponse: service.cancel,
now: () => 6000,
}),
cancel,
revoke: () => {
authorized = false;
},
};
}
async function publish(response: Response) {
const result = await storage.publish(owner, stored.responseId, stored.revision, {
status: response.status,
responseJSON: JSON.stringify(response),
generationCreatedAt: 1234,
});
if (!result.ok) throw new Error('Publication failed');
stored = result.record;
}
it('carries a nonterminal recovery warning without publishing stale actions, output or enqueueing work', async () => {
await scoped(async () => {
await publish({
...buildPendingBackgroundResponse(stored),
status: 'in_progress',
recovery: 'execution_state_unavailable',
});
const f = fixture();
const result = await f.project(run);
expect(result.snapshot).toMatchObject({
status: 'running',
text: '',
recovery: 'execution_state_unavailable',
});
expect(result.snapshot.pendingAction).toBeUndefined();
expect(f.cancel).not.toHaveBeenCalled();
});
});
it('projects only the canonical final assistant text and actual usage', () =>
scoped(async () => {
const f = fixture();
const response: Response = {
...buildPendingBackgroundResponse(stored),
status: 'completed',
completed_at: 5,
output: [
{
id: 'message',
type: 'message',
role: 'assistant',
status: 'completed',
content: [{ type: 'output_text', text: '# Final answer', annotations: [], logprobs: [] }],
},
{
id: 'call',
type: 'function_call',
call_id: 'tool',
name: 'private_tool',
arguments: 'PRIVATE arguments',
status: 'completed',
},
],
usage: {
input_tokens: 10,
output_tokens: 2,
total_tokens: 12,
input_tokens_details: { cached_tokens: 3 },
output_tokens_details: { reasoning_tokens: 1 },
},
};
await publish(response);
const result = await f.project(run);
expect(result).toMatchObject({
generationCreatedAt: 1234,
snapshot: {
responseId: 'resp_saved',
status: 'completed',
text: '# Final answer',
elapsedMs: 4000,
usage: {
inputTokens: 10,
outputTokens: 2,
totalTokens: 12,
cachedTokens: 3,
reasoningTokens: 1,
},
},
});
expect(JSON.stringify(result)).not.toContain('PRIVATE');
expect(f.cancel).not.toHaveBeenCalled();
}));
it('never presents intermediate output as the final Slack answer', () =>
scoped(async () => {
const f = fixture();
await publish({
...buildPendingBackgroundResponse(stored),
status: 'in_progress',
output: [
{
id: 'message',
type: 'message',
role: 'assistant',
status: 'in_progress',
content: [{ type: 'output_text', text: 'Still changing', annotations: [], logprobs: [] }],
},
],
});
expect((await f.project(run)).snapshot).toMatchObject({ status: 'running', text: '' });
}));
it('keeps a cancellation request running until the Responses runtime confirms settlement', () =>
scoped(async () => {
const f = fixture();
await publish({ ...buildPendingBackgroundResponse(stored), status: 'in_progress' });
run.cancellationRequested = true;
expect((await f.project(run)).snapshot.status).toBe('running');
expect(f.cancel).toHaveBeenCalledTimes(1);
await publish({
...buildPendingBackgroundResponse(stored),
status: 'cancelled',
completed_at: 5,
});
expect((await f.project(run)).snapshot.status).toBe('cancelled');
}));
it('honors current source credential authority before returning output', () =>
scoped(async () => {
const f = fixture();
f.revoke();
await expect(f.project(run)).rejects.toMatchObject({ code: 'execution_identity_changed' });
}));
it.each(['responseId', 'conversationId', 'parentMessageId', 'generationCreatedAt'])(
'rejects changed %s rather than reading the latest conversation output',
(field) =>
scoped(async () => {
const f = fixture();
run.execution = {
...run.execution!,
[field]: field === 'generationCreatedAt' ? 5678 : 'other',
};
await expect(f.project(run)).rejects.toMatchObject({ code: 'execution_identity_changed' });
}),
);
it.each(['expired', 'closed'] as const)(
'retires a %s bound Response through cleanup identity without disclosing payload',
(state) =>
scoped(async () => {
run.cancellationRequested = true;
run.errorCode = 'channel_authority_changed';
run.execution!.apiKeyId = 'key';
if (state === 'expired')
await mongoose.models.StoredResponse.updateOne(
{ responseId: stored.responseId },
{ $set: { expiredAt: new Date(0) } },
);
else {
await mongoose.models.Conversation.create({
user: owner.userId,
tenantId: owner.tenantId,
conversationId: stored.conversationId,
endpoint: 'agents',
});
await expect(
createMethods(mongoose).deleteConvos(
owner.userId,
{ conversationId: stored.conversationId },
{
beforeDelete: async () => {
throw new Error('erasure paused after scope closure');
},
},
),
).rejects.toThrow('erasure paused after scope closure');
}
expect(await storage.read(owner, stored.responseId)).toBeNull();
const jobs = {
getJob: jest.fn(async () => undefined),
abortJob: jest.fn(),
finishProviderTerminalPersistence: jest.fn(),
};
const retire = createChannelResponseRetirement({
storage,
recovery: { recoverResponse: jest.fn() },
retireResponse: createResponseRetirement({ storage, jobs }),
});
await expect(retire(run)).resolves.toBe('stopped');
expect(jobs.abortJob).not.toHaveBeenCalled();
}),
);
it.each([
{ requestKey: 'foreign' },
{ sourceKeyId: 'foreign' },
{ parentMessageId: 'foreign' },
{ agentId: 'foreign' },
{ conversationId: 'foreign' },
])('refuses foreign cleanup binding metadata after expiry: %j', (change) =>
scoped(async () => {
run.cancellationRequested = true;
run.errorCode = 'channel_authority_changed';
run.execution!.apiKeyId = 'key';
await mongoose.models.StoredResponse.collection.updateOne(
{ responseId: stored.responseId },
{ $set: { expiredAt: new Date(0), ...change } },
);
const retireResponse = jest.fn(async () => 'stopped' as const);
const retire = createChannelResponseRetirement({
storage,
recovery: { recoverResponse: jest.fn() },
retireResponse,
});
await expect(retire(run)).rejects.toMatchObject({ code: 'execution_identity_changed' });
expect(retireResponse).not.toHaveBeenCalled();
}),
);
it('preserves pending retirement when an expired generation has no drain proof', () =>
scoped(async () => {
run.cancellationRequested = true;
run.errorCode = 'channel_authority_changed';
run.execution!.apiKeyId = 'key';
await mongoose.models.StoredResponse.updateOne(
{ responseId: stored.responseId },
{ $set: { expiredAt: new Date(0), generationCreatedAt: 1234 } },
);
const jobs = {
getJob: jest.fn(async () => undefined),
abortJob: jest.fn(),
finishProviderTerminalPersistence: jest.fn(),
};
const retire = createChannelResponseRetirement({
storage,
recovery: { recoverResponse: jest.fn() },
retireResponse: createResponseRetirement({ storage, jobs }),
});
await expect(retire(run)).resolves.toBe('pending');
expect(jobs.abortJob).not.toHaveBeenCalled();
}));
it('does not treat a cleanup metadata outage as successful retirement', () =>
scoped(async () => {
run.cancellationRequested = true;
run.errorCode = 'channel_authority_changed';
const retireResponse = jest.fn(async () => 'stopped' as const);
const retire = createChannelResponseRetirement({
storage: {
readCleanupIdentity: async () => {
throw new Error('cleanup store unavailable');
},
},
recovery: { recoverResponse: jest.fn() },
retireResponse,
});
await expect(retire(run)).rejects.toThrow('cleanup store unavailable');
expect(retireResponse).not.toHaveBeenCalled();
}));

View file

@ -0,0 +1,167 @@
import type {
ChannelRunRecord,
ChannelRunStorage,
ResponseStorage,
ResponseOwner,
} from '@librechat/data-schemas';
import type { ChannelRunSnapshot } from 'librechat-data-provider';
import type { BackgroundResponseResult } from '~/agents/responses/background';
import type { Response } from '~/agents/responses/types';
import { ChannelExecutionError } from './errors';
export interface ChannelResponseProjectionDependencies {
storage: Pick<ResponseStorage, 'read'>;
/** Reads through live source-key, owner, tenant, role and Agent authorization. */
readResponse: (owner: ResponseOwner, responseId: string) => Promise<BackgroundResponseResult>;
cancelResponse: (owner: ResponseOwner, responseId: string) => Promise<BackgroundResponseResult>;
now?: () => number;
}
/** Revoked channel authority can retire its exact response without recovering execution authority. */
export function createChannelResponseRetirement(deps: {
storage: Pick<ResponseStorage, 'readCleanupIdentity'>;
recovery: Pick<ChannelRunStorage, 'recoverResponse'>;
retireResponse: (owner: ResponseOwner, responseId: string) => Promise<'stopped' | 'pending'>;
now?: () => number;
}): (run: ChannelRunRecord) => Promise<'stopped' | 'pending'> {
return async (run) => {
if (
run.execution?.protocol !== 'responses' ||
!run.cancellationRequested ||
run.errorCode !== 'channel_authority_changed'
)
throw new ChannelExecutionError('execution_identity_changed');
if (!run.execution.responseId) {
if (!run.leaseOwner) throw new ChannelExecutionError('execution_identity_changed');
const recovered = await deps.recovery.recoverResponse(
run.tenantId,
{ runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch },
(deps.now ?? Date.now)(),
);
if (!recovered) return 'pending';
run = recovered;
}
const responseId = run.execution?.responseId;
if (!responseId) return 'pending';
const owner = {
userId: run.principal.userId,
tenantId: run.tenantId === '_' ? undefined : run.tenantId,
};
const stored = await deps.storage.readCleanupIdentity(owner, responseId);
if (
stored &&
(stored.responseId !== responseId ||
stored.userId !== owner.userId ||
(stored.tenantId || undefined) !== (owner.tenantId || undefined) ||
stored.agentId !== run.agentId ||
stored.requestKey !== `channel:${run.runId}` ||
stored.conversationId !== run.execution!.conversationId ||
stored.parentMessageId !== run.execution!.parentMessageId ||
(run.execution!.apiKeyId != null && stored.sourceKeyId !== run.execution!.apiKeyId))
)
throw new ChannelExecutionError('execution_identity_changed');
// The private retirement owner resolves missing rows from cleanup/job identities;
// absence here never proves that a provider stopped.
return deps.retireResponse(owner, responseId);
};
}
function snapshotStatus(response: Response): ChannelRunSnapshot['status'] {
switch (response.status) {
case 'completed':
return 'completed';
case 'cancelled':
return 'cancelled';
case 'failed':
case 'incomplete':
return 'failed';
default:
return response.required_action ? 'requires_action' : 'running';
}
}
/** Delivery projection only: never starts an Agent, rebuilds a transcript or infers completion. */
export function createChannelResponseProjection(deps: ChannelResponseProjectionDependencies) {
return async (
run: ChannelRunRecord,
): Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }> => {
const execution = run.execution;
if (execution?.protocol !== 'responses' || !execution.responseId)
throw new ChannelExecutionError('execution_not_prepared');
const owner = {
userId: run.principal.userId,
tenantId: run.tenantId === '_' ? undefined : run.tenantId,
};
const stored = await deps.storage.read(owner, execution.responseId);
if (
!stored ||
stored.agentId !== run.agentId ||
stored.requestKey !== `channel:${run.runId}` ||
stored.conversationId !== execution.conversationId ||
stored.parentMessageId !== execution.parentMessageId ||
(execution.apiKeyId != null && stored.sourceKeyId !== execution.apiKeyId) ||
(execution.generationCreatedAt != null &&
stored.generationCreatedAt !== execution.generationCreatedAt)
)
throw new ChannelExecutionError('execution_identity_changed');
const result = await (run.cancellationRequested ? deps.cancelResponse : deps.readResponse)(
owner,
execution.responseId,
);
if (!result.ok) throw new ChannelExecutionError('execution_identity_changed');
const response = result.response;
if (
response.id !== execution.responseId ||
response.model !== run.agentId ||
!response.background ||
!response.store
)
throw new ChannelExecutionError('execution_identity_changed');
const text: string[] = [];
for (const item of response.output) {
if (item.type !== 'message' || item.role !== 'assistant') continue;
for (const content of item.content) {
if (content.type === 'output_text') text.push(content.text);
}
}
const terminal = ['completed', 'failed', 'incomplete', 'cancelled'].includes(response.status);
const status = snapshotStatus(response);
const action = response.required_action;
return {
generationCreatedAt: stored.generationCreatedAt,
snapshot: {
responseId: response.id,
status,
...(response.recovery ? { recovery: response.recovery } : {}),
text: terminal ? text.join('') : '',
elapsedMs: Math.max(
0,
(response.completed_at == null
? (deps.now ?? Date.now)()
: response.completed_at * 1000) - run.receivedAt,
),
...(response.accounting ? { accounting: response.accounting } : {}),
...(response.usage
? {
usage: {
inputTokens: response.usage.input_tokens,
outputTokens: response.usage.output_tokens,
totalTokens: response.usage.total_tokens,
cachedTokens: response.usage.input_tokens_details.cached_tokens,
reasoningTokens: response.usage.output_tokens_details.reasoning_tokens,
},
}
: {}),
...(!terminal && action
? {
pendingAction: {
actionId: action.action_id,
kind: action.kind === 'question' ? ('ask_user_question' as const) : action.kind,
expiresAt: action.expires_at * 1000,
},
}
: {}),
},
};
};
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -158,12 +158,14 @@ export * from './mcp/approval';
export { createChannelStorageConfigReader } from './channels/config';
export * from './channels/service';
export * from './channels/runs';
export * from './channels/authority';
export * from './channels/execution';
export * from './channels/responses';
export * from './channels/credentials';
export * from './channels/identity';
export * from './channels/identityHttp';
export * from './channels/artifacts';
export * from './channels/artifactStorage';
export * from './channels/cleanup';
export * from './channels/summary';

View file

@ -111,6 +111,8 @@ export interface ChannelDeliveryPart {
text: string;
/** Provider-defined, versioned JSON presentation; immutable alongside the text fallback. */
presentation?: { format: string; payload: string };
/** Optional diagnostic operation; reauthorized independently of immutable answer delivery. */
traceUrl?: string;
/** Opaque server-authorized artifact identity; never an arbitrary URL or disk path. */
artifactId?: string;
}

View file

@ -187,6 +187,7 @@ export function assertPlan(
(part) =>
!part.operationId ||
!['message', 'artifact'].includes(part.kind) ||
(part.traceUrl != null && (part.required || part.kind !== 'message' || !part.traceUrl)) ||
(part.kind === 'artifact' &&
(!part.artifactId ||
!run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))),

View file

@ -2193,3 +2193,23 @@ it('closes and cooperatively cancels private prepared work while retaining redac
await eraseResponseRecords(mongoose, owner, ['conversation-one']);
expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0);
}));
it.each(['live', 'expired', 'closed'] as const)(
'returns immutable cleanup binding metadata without payload for %s rows',
(state) =>
scoped(async () => {
await storage.reserve(reservation({ sourceKeyId: 'run-key' }));
if (state === 'expired')
await createResponseRetention(mongoose)(owner, 'conversation-one', new Date(0));
if (state === 'closed') await closeResponseScopes(mongoose, owner, ['conversation-one']);
const identity = await storage.readCleanupIdentity(owner, 'resp_one');
expect(identity).toMatchObject({
requestKey: 'request-one',
sourceKeyId: 'run-key',
parentMessageId: 'root',
});
expect(identity).not.toHaveProperty('requestJSON');
expect(identity).not.toHaveProperty('responseJSON');
expect(JSON.stringify(identity)).not.toContain('Hello');
}),
);

View file

@ -52,7 +52,8 @@ export type ResponseMutationResult =
| { ok: true; record: StoredResponseRecord }
| { ok: false; code: 'response_conflict' };
/** Internal drain identity only; expired requests, answers and decisions are never returned. */
/** Internal drain identity only; expired requests, answers and decisions are never returned.
* Stored rows include immutable request binding; older retained job identities may omit it. */
export type ResponseCleanupIdentity = Pick<
StoredResponseRecord,
| 'userId'
@ -62,13 +63,14 @@ export type ResponseCleanupIdentity = Pick<
| 'conversationId'
| 'generationCreatedAt'
| 'expiredAt'
> & {
/** Cleanup authority only; never permission to read output or execute a provider. */
retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired';
checkpointNamespace?: string;
checkpointConfigJSON?: string;
requiresCheckpointCleanup?: boolean;
};
> &
Partial<Pick<StoredResponseRecord, 'requestKey' | 'sourceKeyId' | 'parentMessageId'>> & {
/** Cleanup authority only; never permission to read output or execute a provider. */
retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired';
checkpointNamespace?: string;
checkpointConfigJSON?: string;
requiresCheckpointCleanup?: boolean;
};
export interface ResponseResumeReservation
extends Omit<ResponseResumeIntent, 'state' | 'reservedAt' | 'admittedAt'> {
@ -851,7 +853,7 @@ export function createResponseStorage(
const record = await model
.findOne({ ...ownerScope(owner), responseId })
.select(
'userId tenantId responseId agentId conversationId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON preparedTerminal.checkpointNamespace preparedTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId',
'userId tenantId responseId agentId conversationId requestKey sourceKeyId parentMessageId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON preparedTerminal.checkpointNamespace preparedTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId',
)
.lean();
// Retained data comes only from an internal job/receipt, never request input.
@ -928,6 +930,9 @@ export function createResponseStorage(
responseId: id,
agentId,
conversationId,
...(source.requestKey == null ? {} : { requestKey: source.requestKey }),
...(source.sourceKeyId == null ? {} : { sourceKeyId: source.sourceKeyId }),
...(source.parentMessageId == null ? {} : { parentMessageId: source.parentMessageId }),
generationCreatedAt:
generationCreatedAt ?? (retirement ? matchingRetained?.generationCreatedAt : undefined),
expiredAt,