mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-21 23:43:35 +00:00
📨 feat: Coordinate Channel Delivery Through Responses (#17609)
* 📨 feat: Coordinate Channel Delivery Through Responses
* fix: Enforce Channel Credential Retirement and Live Trace Delivery
* test: derive artifact policy fixtures from validated schema
* Enforce Current Channel Credentials and Recover Worker Initialization
* Complete Channel Capability and Durable Reconciliation Boundaries
This commit is contained in:
parent
aae13f6817
commit
c907dbf346
17 changed files with 5773 additions and 12 deletions
1550
packages/api/src/channels/authority.spec.ts
Normal file
1550
packages/api/src/channels/authority.spec.ts
Normal file
File diff suppressed because it is too large
Load diff
629
packages/api/src/channels/cleanup.spec.ts
Normal file
629
packages/api/src/channels/cleanup.spec.ts
Normal file
|
|
@ -0,0 +1,629 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryReplSet } from 'mongodb-memory-server';
|
||||
import { channelConfigSchema } from 'librechat-data-provider';
|
||||
import {
|
||||
createModels,
|
||||
createMethods,
|
||||
createChannelRunStorage,
|
||||
createResponseStorage,
|
||||
runAsSystem,
|
||||
tenantStorage,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { ChannelRunRecord } from '@librechat/data-schemas';
|
||||
import type { ChannelCleanupDependencies } from './cleanup';
|
||||
import { createResponseRetirement } from '~/agents/responses/retirement';
|
||||
import { createChannelResponseRetirement } from './responses';
|
||||
import { createChannelExecutionAdapter } from './execution';
|
||||
import { createChannelCleanupRuntime } from './cleanup';
|
||||
import * as recovery from '~/agents/recovery';
|
||||
|
||||
let mongo: MongoMemoryReplSet;
|
||||
let db: ReturnType<typeof createMethods>;
|
||||
let storage: ReturnType<typeof createChannelRunStorage>;
|
||||
let deps: ChannelCleanupDependencies;
|
||||
let runtime: ReturnType<typeof createChannelCleanupRuntime>;
|
||||
let retired: ChannelRunRecord[];
|
||||
let allowed: boolean;
|
||||
let enabled: boolean;
|
||||
let stoppedResult: 'stopped' | 'pending';
|
||||
let clock: number;
|
||||
const tenantId = 'cleanup-tenant';
|
||||
const scope = <T>(work: () => Promise<T>) => tenantStorage.run({ tenantId }, work);
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } });
|
||||
await mongoose.connect(mongo.getUri());
|
||||
createModels(mongoose);
|
||||
db = createMethods(mongoose);
|
||||
storage = createChannelRunStorage(mongoose);
|
||||
await Promise.all([mongoose.models.ChannelRun.init(), mongoose.models.ChannelLane.init()]);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongo?.stop();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await runAsSystem(() =>
|
||||
Promise.all(
|
||||
[
|
||||
'ChannelRun',
|
||||
'ChannelLane',
|
||||
'AgentChannel',
|
||||
'ChannelIdentity',
|
||||
'ChannelScope',
|
||||
'ResponseScope',
|
||||
'StoredResponse',
|
||||
].map((name) => mongoose.models[name].deleteMany({})),
|
||||
),
|
||||
);
|
||||
clock = Date.now();
|
||||
allowed = true;
|
||||
enabled = true;
|
||||
retired = [];
|
||||
stoppedResult = 'stopped';
|
||||
await scope(() =>
|
||||
db.createAgentChannel({
|
||||
channelId: 'channel',
|
||||
tenantId,
|
||||
agentId: 'agent',
|
||||
createdBy: 'user-a',
|
||||
type: 'slack',
|
||||
credentialMode: 'per_user',
|
||||
status: 'active',
|
||||
revision: 'v1',
|
||||
encryptedBotToken: 'not-a-real-token',
|
||||
slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' },
|
||||
}),
|
||||
);
|
||||
await scope(() =>
|
||||
mongoose.models.ChannelIdentity.create({
|
||||
tenantId,
|
||||
linkId: 'link',
|
||||
revision: 'link-v1',
|
||||
state: 'active',
|
||||
userId: 'user-a',
|
||||
ticketHash: 'ticket',
|
||||
ticketExpiresAtMs: clock + 60000,
|
||||
expiresAtMs: clock + 600000,
|
||||
identity: {
|
||||
tenantId,
|
||||
channelId: 'channel',
|
||||
channelRevision: 'v1',
|
||||
teamId: 'T1',
|
||||
externalUserId: 'U1',
|
||||
agentId: 'agent',
|
||||
},
|
||||
}),
|
||||
);
|
||||
deps = {
|
||||
storage,
|
||||
isCredentialCurrent: async () => true,
|
||||
channels: db,
|
||||
now: () => clock,
|
||||
getConfig: async () =>
|
||||
channelConfigSchema.parse({
|
||||
enabled,
|
||||
cleanup: { intervalMs: 60_000, maxIdleIntervalMs: 60_000, batchSize: 2 },
|
||||
}),
|
||||
resolvePrincipal: async () =>
|
||||
allowed
|
||||
? { userId: 'user-a', channelRevision: 'v1', linkId: 'link', linkRevision: 'link-v1' }
|
||||
: null,
|
||||
retireForAuthorityLoss: async (run) => {
|
||||
expect(tenantStorage.getStore()?.tenantId).toBe(
|
||||
run.tenantId === '_' ? undefined : run.tenantId,
|
||||
);
|
||||
retired.push(run);
|
||||
return stoppedResult;
|
||||
},
|
||||
deleteApiKey: async () => true,
|
||||
onError: jest.fn(),
|
||||
};
|
||||
runtime = createChannelCleanupRuntime(deps);
|
||||
});
|
||||
afterEach(async () => {
|
||||
await runtime.stop();
|
||||
});
|
||||
async function admit(eventId = 'event') {
|
||||
return scope(() =>
|
||||
storage.admit(
|
||||
{
|
||||
tenantId,
|
||||
channelId: 'channel',
|
||||
agentId: 'agent',
|
||||
principal: {
|
||||
userId: 'user-a',
|
||||
channelRevision: 'v1',
|
||||
linkId: 'link',
|
||||
linkRevision: 'link-v1',
|
||||
},
|
||||
admission: {
|
||||
eventId,
|
||||
externalChannelId: 'C1',
|
||||
externalThreadKey: 'C1:1',
|
||||
externalMessageKey: `C1:${eventId}`,
|
||||
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
|
||||
input: 'private input',
|
||||
},
|
||||
},
|
||||
clock,
|
||||
),
|
||||
);
|
||||
}
|
||||
async function restart() {
|
||||
await runtime.stop();
|
||||
runtime = createChannelCleanupRuntime(deps);
|
||||
await runtime.initialize();
|
||||
}
|
||||
|
||||
it('does not start by construction, initializes once, and leaves authorized active work untouched', async () => {
|
||||
const { run } = await admit();
|
||||
expect(retired).toHaveLength(0);
|
||||
await Promise.all([runtime.initialize(), runtime.initialize()]);
|
||||
expect(retired).toHaveLength(0);
|
||||
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
|
||||
state: 'queued',
|
||||
leaseEpoch: 0,
|
||||
cancellationRequested: false,
|
||||
});
|
||||
await runtime.stop();
|
||||
allowed = false;
|
||||
await runtime.initialize();
|
||||
expect(retired).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(['revoked', 'disabled', 'feature-off', 'orphan'] as const)(
|
||||
'cleans %s work without a gateway or any credential bundle',
|
||||
async (reason) => {
|
||||
const { run } = await admit();
|
||||
await scope(() => storage.claim(tenantId, 'channel', 'absent-gateway', clock, 60_000));
|
||||
if (reason === 'revoked') allowed = false;
|
||||
if (reason === 'disabled')
|
||||
await scope(() => db.updateAgentChannel('channel', 'user-a', { status: 'disabled' }));
|
||||
if (reason === 'feature-off') enabled = false;
|
||||
if (reason === 'orphan')
|
||||
await scope(() => mongoose.models.AgentChannel.deleteOne({ channelId: 'channel' }));
|
||||
const bundles = jest.spyOn(db, 'getAgentChannel');
|
||||
await runtime.initialize();
|
||||
expect(bundles).not.toHaveBeenCalled();
|
||||
expect(retired).toHaveLength(1);
|
||||
expect(retired[0]).toMatchObject({ runId: run.runId, tenantId, principal: run.principal });
|
||||
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
|
||||
state: 'cancelled',
|
||||
errorCode: 'channel_authority_changed',
|
||||
});
|
||||
expect(deps.onError).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('defers deletion while cleanup is unknown and completes after restart and exact lease expiry', async () => {
|
||||
const { run } = await admit();
|
||||
await scope(() => db.updateAgentChannel('channel', 'user-a', { status: 'deleting' }));
|
||||
expect(await scope(() => db.deleteAgentChannel('channel', 'user-a'))).toEqual({
|
||||
status: 'pending_drain',
|
||||
});
|
||||
stoppedResult = 'pending';
|
||||
await runtime.initialize();
|
||||
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toMatchObject({
|
||||
status: 'deleting',
|
||||
});
|
||||
await restart();
|
||||
expect(retired).toHaveLength(1);
|
||||
clock += 60_001;
|
||||
stoppedResult = 'stopped';
|
||||
await restart();
|
||||
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toBeNull();
|
||||
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
|
||||
state: 'cancelled',
|
||||
});
|
||||
});
|
||||
|
||||
it('recovers deleting parents with no remaining runs but never bypasses a failed legacy key revocation', async () => {
|
||||
await scope(() => db.updateAgentChannel('channel', 'user-a', { apiKeyId: 'legacy-key-id' }));
|
||||
await scope(() => db.beginAgentChannelDeletion('channel', 'user-a'));
|
||||
deps.deleteApiKey = async () => {
|
||||
throw new Error('key store unavailable');
|
||||
};
|
||||
await runtime.initialize();
|
||||
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toMatchObject({
|
||||
status: 'deleting',
|
||||
});
|
||||
deps.deleteApiKey = async () => true;
|
||||
await restart();
|
||||
expect(await scope(() => db.getAgentChannelMetadata('channel'))).toBeNull();
|
||||
});
|
||||
|
||||
it('does not convert authority lookup outages into cleanup', async () => {
|
||||
const { run } = await admit();
|
||||
deps.resolvePrincipal = async () => {
|
||||
throw new Error('permissions unavailable');
|
||||
};
|
||||
await runtime.initialize();
|
||||
expect(retired).toHaveLength(0);
|
||||
expect(await scope(() => storage.read(tenantId, 'channel', run.runId))).toMatchObject({
|
||||
state: 'queued',
|
||||
cancellationRequested: false,
|
||||
leaseEpoch: 0,
|
||||
});
|
||||
expect(deps.onError).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('enumerates only bounded metadata in system context then scopes every full read to its tenant', async () => {
|
||||
await admit('one');
|
||||
await admit('two');
|
||||
await admit('three');
|
||||
const metadata = await runAsSystem(() => storage.listPendingMetadata(undefined, 2));
|
||||
expect(metadata.runs).toHaveLength(2);
|
||||
expect(metadata.next).toBeDefined();
|
||||
expect(Object.keys(metadata.runs[0]).sort()).toEqual(['channelId', 'runId', 'tenantId']);
|
||||
const read = jest.spyOn(storage, 'read');
|
||||
allowed = false;
|
||||
await runtime.initialize();
|
||||
expect(retired).toHaveLength(2);
|
||||
expect(read.mock.calls.every(([tenant]) => tenant === tenantId)).toBe(true);
|
||||
expect(await scope(() => db.getAgentChannelMetadata('channel'))).not.toHaveProperty(
|
||||
'encryptedBotToken',
|
||||
);
|
||||
});
|
||||
|
||||
it('waits for owned cleanup on shutdown and does not start the next run', async () => {
|
||||
await admit('one');
|
||||
await admit('two');
|
||||
allowed = false;
|
||||
let entered!: () => void;
|
||||
let release!: () => void;
|
||||
const ready = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
deps.retireForAuthorityLoss = async (run) => {
|
||||
retired.push(run);
|
||||
entered();
|
||||
await gate;
|
||||
return 'stopped';
|
||||
};
|
||||
const initializing = runtime.initialize();
|
||||
await ready;
|
||||
let ended = false;
|
||||
const stopping = runtime.stop().then(() => {
|
||||
ended = true;
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(ended).toBe(false);
|
||||
release();
|
||||
await initializing;
|
||||
await stopping;
|
||||
expect(retired).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('uses an allowlist so future top-level or Slack credential fields cannot enter authority metadata', async () => {
|
||||
await mongoose.models.AgentChannel.collection.updateOne(
|
||||
{ tenantId, channelId: 'channel' },
|
||||
{ $set: { futureCredential: 'must-not-leak', 'slack.futureCredential': 'must-not-leak' } },
|
||||
);
|
||||
const metadata = await scope(() => db.getAgentChannelMetadata('channel'));
|
||||
expect(Object.keys(metadata!).sort()).toEqual([
|
||||
'agentId',
|
||||
'channelId',
|
||||
'createdBy',
|
||||
'credentialMode',
|
||||
'revision',
|
||||
'slack',
|
||||
'status',
|
||||
'tenantId',
|
||||
'type',
|
||||
]);
|
||||
expect(metadata?.slack).toEqual({ teamId: 'T1', appId: 'A1', botUserId: 'UBOT' });
|
||||
expect(JSON.stringify(metadata)).not.toContain('must-not-leak');
|
||||
});
|
||||
|
||||
it('defines bounded cleanup configuration without exposing the algorithmic claim batch as a policy knob', () => {
|
||||
const config = channelConfigSchema.parse({});
|
||||
expect(config.cleanup).toEqual({ batchSize: 25, intervalMs: 1000, maxIdleIntervalMs: 30_000 });
|
||||
expect(config.claimBatchSize).toBe(25);
|
||||
for (const cleanup of [
|
||||
{ batchSize: 0 },
|
||||
{ batchSize: 101 },
|
||||
{ intervalMs: 999 },
|
||||
{ maxIdleIntervalMs: 300_001 },
|
||||
]) {
|
||||
expect(channelConfigSchema.safeParse({ cleanup }).success).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps tenantless work authorized and deletes its closed parent in the application default tenant', async () => {
|
||||
const defaultScope = <T>(work: () => Promise<T>) =>
|
||||
tenantStorage.run({ tenantId: undefined }, work);
|
||||
await defaultScope(async () => {
|
||||
await db.createAgentChannel({
|
||||
channelId: 'default-channel',
|
||||
agentId: 'agent',
|
||||
createdBy: 'user-a',
|
||||
type: 'slack',
|
||||
credentialMode: 'per_user',
|
||||
status: 'active',
|
||||
revision: 'v1',
|
||||
slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' },
|
||||
});
|
||||
await mongoose.models.ChannelIdentity.create({
|
||||
tenantId: '_',
|
||||
linkId: 'link',
|
||||
revision: 'link-v1',
|
||||
state: 'active',
|
||||
userId: 'user-a',
|
||||
ticketHash: 'default-ticket',
|
||||
ticketExpiresAtMs: clock + 60000,
|
||||
expiresAtMs: clock + 600000,
|
||||
identity: {
|
||||
tenantId: '_',
|
||||
channelId: 'default-channel',
|
||||
channelRevision: 'v1',
|
||||
teamId: 'T1',
|
||||
externalUserId: 'U1',
|
||||
agentId: 'agent',
|
||||
},
|
||||
});
|
||||
await storage.admit(
|
||||
{
|
||||
tenantId: '_',
|
||||
channelId: 'default-channel',
|
||||
agentId: 'agent',
|
||||
principal: {
|
||||
userId: 'user-a',
|
||||
channelRevision: 'v1',
|
||||
linkId: 'link',
|
||||
linkRevision: 'link-v1',
|
||||
},
|
||||
admission: {
|
||||
eventId: 'default-event',
|
||||
externalChannelId: 'C1',
|
||||
externalThreadKey: 'C1:1',
|
||||
externalMessageKey: 'C1:1',
|
||||
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
|
||||
input: 'default tenant input',
|
||||
},
|
||||
},
|
||||
clock,
|
||||
);
|
||||
});
|
||||
await runtime.initialize();
|
||||
expect(retired).toHaveLength(0);
|
||||
expect(deps.onError).not.toHaveBeenCalled();
|
||||
await defaultScope(() => db.beginAgentChannelDeletion('default-channel', 'user-a'));
|
||||
await restart();
|
||||
expect(retired).toHaveLength(1);
|
||||
expect(retired[0].tenantId).toBe('_');
|
||||
expect(await defaultScope(() => db.getAgentChannelMetadata('default-channel'))).toBeNull();
|
||||
expect(deps.onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('caps maintenance pages by the storage policy and never turns a credential lookup outage into retirement', async () => {
|
||||
await admit('one');
|
||||
await admit('two');
|
||||
deps.getConfig = async () =>
|
||||
channelConfigSchema.parse({
|
||||
enabled: true,
|
||||
maxMetadataPageSize: 1,
|
||||
cleanup: { batchSize: 2, intervalMs: 60000, maxIdleIntervalMs: 60000 },
|
||||
});
|
||||
const read = jest.spyOn(storage, 'listPendingMetadata');
|
||||
deps.isCredentialCurrent = async () => {
|
||||
throw new Error('credential store unavailable');
|
||||
};
|
||||
await runtime.initialize();
|
||||
expect(read).toHaveBeenCalledWith(undefined, 1);
|
||||
expect(retired).toHaveLength(0);
|
||||
expect(deps.onError).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('holds an invalid issued credential lane until exact retirement confirms provider drain', async () => {
|
||||
const { run } = await admit('expired-credential');
|
||||
const claimed = (await scope(() => storage.claim(tenantId, 'channel', 'gateway', clock, 60000)))!;
|
||||
const fence = { runId: run.runId, owner: 'gateway', epoch: claimed.leaseEpoch };
|
||||
await scope(() => storage.prepareExecution(tenantId, fence, clock));
|
||||
await scope(() =>
|
||||
storage.attachResponseCredential(
|
||||
tenantId,
|
||||
fence,
|
||||
{ apiKeyId: 'expired-key', encryptedApiKey: 'encrypted-original' },
|
||||
clock,
|
||||
),
|
||||
);
|
||||
deps.isCredentialCurrent = async (current) => {
|
||||
expect(current.execution?.apiKeyId).toBe('expired-key');
|
||||
return false;
|
||||
};
|
||||
stoppedResult = 'pending';
|
||||
await runtime.initialize();
|
||||
expect(retired).toHaveLength(1);
|
||||
expect(retired[0].execution?.apiKeyId).toBe('expired-key');
|
||||
expect(
|
||||
await scope(() => mongoose.models.ChannelLane.findOne({ activeRunId: run.runId }).lean()),
|
||||
).not.toBeNull();
|
||||
expect((await scope(() => storage.read(tenantId, 'channel', run.runId)))?.state).not.toBe(
|
||||
'cancelled',
|
||||
);
|
||||
clock += 60001;
|
||||
stoppedResult = 'stopped';
|
||||
await restart();
|
||||
expect(retired).toHaveLength(2);
|
||||
expect((await scope(() => storage.read(tenantId, 'channel', run.runId)))?.state).toBe(
|
||||
'cancelled',
|
||||
);
|
||||
expect(
|
||||
await scope(() => mongoose.models.ChannelLane.findOne({ activeRunId: run.runId }).lean()),
|
||||
).toBeNull();
|
||||
expect(deps.onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['read', 'parse'] as const)(
|
||||
'retries failed %s initialization with one shared attempt',
|
||||
async (failure) => {
|
||||
await admit();
|
||||
allowed = false;
|
||||
const original = deps.getConfig;
|
||||
let fail = true;
|
||||
const read = jest.fn(async () => {
|
||||
if (fail) {
|
||||
if (failure === 'read') throw new Error('configuration unavailable');
|
||||
const invalid = channelConfigSchema.parse({});
|
||||
invalid.cleanup.intervalMs = 0;
|
||||
return invalid;
|
||||
}
|
||||
return original();
|
||||
});
|
||||
deps.getConfig = read;
|
||||
const first = runtime.initialize();
|
||||
expect(runtime.initialize()).toBe(first);
|
||||
await expect(first).rejects.toThrow();
|
||||
fail = false;
|
||||
const retry = runtime.initialize();
|
||||
expect(runtime.initialize()).toBe(retry);
|
||||
await retry;
|
||||
expect(retired).toHaveLength(1);
|
||||
const calls = read.mock.calls.length;
|
||||
await runtime.initialize();
|
||||
expect(read).toHaveBeenCalledTimes(calls);
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps stop permanent after an initialization failure', async () => {
|
||||
const read = jest.fn(async () => {
|
||||
throw new Error('configuration unavailable');
|
||||
});
|
||||
deps.getConfig = read;
|
||||
await expect(runtime.initialize()).rejects.toThrow('configuration unavailable');
|
||||
await runtime.stop();
|
||||
await runtime.initialize();
|
||||
runtime.wake();
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
expect(retired).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('disposes a partially started worker before retrying initialization', async () => {
|
||||
await admit();
|
||||
allowed = false;
|
||||
const create = recovery.createIdleRecoveryLoop;
|
||||
const partial = create({
|
||||
intervalMs: 60000,
|
||||
maxIdleIntervalMs: 60000,
|
||||
scan: async () => true,
|
||||
onError: jest.fn(),
|
||||
});
|
||||
const start = jest
|
||||
.spyOn(partial, 'start')
|
||||
.mockRejectedValueOnce(new Error('initial scan failed'));
|
||||
const stop = jest.spyOn(partial, 'stop');
|
||||
const factory = jest.spyOn(recovery, 'createIdleRecoveryLoop').mockReturnValueOnce(partial);
|
||||
try {
|
||||
await expect(runtime.initialize()).rejects.toThrow('initial scan failed');
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(stop).toHaveBeenCalledTimes(1);
|
||||
await runtime.initialize();
|
||||
expect(retired).toHaveLength(1);
|
||||
expect(factory).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
factory.mockRestore();
|
||||
start.mockRestore();
|
||||
stop.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not reopen when a pending initialization rejects during shutdown', async () => {
|
||||
let reject!: (error: Error) => void;
|
||||
const gate = new Promise<Awaited<ReturnType<typeof deps.getConfig>>>((_resolve, fail) => {
|
||||
reject = fail;
|
||||
});
|
||||
const read = jest.fn(() => gate);
|
||||
deps.getConfig = read;
|
||||
const initializing = runtime.initialize();
|
||||
const stopping = runtime.stop();
|
||||
reject(new Error('startup unavailable'));
|
||||
await expect(initializing).rejects.toThrow('startup unavailable');
|
||||
await stopping;
|
||||
await runtime.initialize();
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
expect(retired).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'releases an expired Response lane only after canonical retirement proof: pending=%s',
|
||||
async (pending) => {
|
||||
await admit();
|
||||
const claimed = (await scope(() =>
|
||||
storage.claim(tenantId, 'channel', 'worker', clock, 60000),
|
||||
))!;
|
||||
const fence = { runId: claimed.runId, owner: claimed.leaseOwner!, epoch: claimed.leaseEpoch };
|
||||
const prepared = await scope(() => storage.prepareExecution(tenantId, fence, clock));
|
||||
await scope(() =>
|
||||
storage.attachResponseCredential(
|
||||
tenantId,
|
||||
fence,
|
||||
{ apiKeyId: 'issued', encryptedApiKey: 'encrypted' },
|
||||
clock,
|
||||
),
|
||||
);
|
||||
const responses = createResponseStorage(mongoose);
|
||||
const execution = prepared.execution!;
|
||||
const owner = { userId: prepared.principal.userId, tenantId };
|
||||
const responseId = `resp_${prepared.runId}`;
|
||||
const reservation = await scope(() =>
|
||||
responses.reserve({
|
||||
...owner,
|
||||
responseId,
|
||||
requestKey: `channel:${prepared.runId}`,
|
||||
sourceKeyId: 'issued',
|
||||
fingerprint: 'cleanup',
|
||||
agentId: prepared.agentId,
|
||||
requestJSON: JSON.stringify({
|
||||
model: prepared.agentId,
|
||||
input: prepared.admission.input,
|
||||
background: true,
|
||||
store: true,
|
||||
...(execution.parentMessageId === '00000000-0000-0000-0000-000000000000'
|
||||
? {}
|
||||
: { previous_response_id: execution.parentMessageId }),
|
||||
}),
|
||||
conversationId: execution.conversationId,
|
||||
parentMessageId: execution.parentMessageId,
|
||||
}),
|
||||
);
|
||||
if (!reservation.ok) throw new Error('Reservation failed');
|
||||
await scope(() => storage.bindResponse(tenantId, fence, responseId, clock));
|
||||
await scope(() =>
|
||||
mongoose.models.StoredResponse.updateOne(
|
||||
{ responseId },
|
||||
{ $set: { expiredAt: new Date(0), ...(pending ? { generationCreatedAt: 1234 } : {}) } },
|
||||
),
|
||||
);
|
||||
const jobs = {
|
||||
getJob: jest.fn(async () => undefined),
|
||||
abortJob: jest.fn(),
|
||||
finishProviderTerminalPersistence: jest.fn(),
|
||||
};
|
||||
const adapter = createChannelExecutionAdapter({
|
||||
responseProjection: async () => {
|
||||
throw new Error('Cleanup must not project or execute');
|
||||
},
|
||||
responseRetirement: createChannelResponseRetirement({
|
||||
storage: responses,
|
||||
recovery: storage,
|
||||
retireResponse: createResponseRetirement({ storage: responses, jobs }),
|
||||
}),
|
||||
});
|
||||
deps.retireForAuthorityLoss = adapter.retireForAuthorityLoss;
|
||||
deps.isCredentialCurrent = async () => false;
|
||||
await runtime.initialize();
|
||||
await runtime.stop();
|
||||
const next = await admit('next');
|
||||
const nextClaim = await scope(() =>
|
||||
storage.claim(tenantId, 'channel', 'next-worker', clock + 60001, 60000),
|
||||
);
|
||||
if (pending) {
|
||||
expect(nextClaim?.runId).toBe(claimed.runId);
|
||||
expect(nextClaim?.errorCode).toBe('channel_authority_changed');
|
||||
} else expect(nextClaim?.runId).toBe(next.run.runId);
|
||||
expect(jobs.abortJob).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
155
packages/api/src/channels/cleanup.ts
Normal file
155
packages/api/src/channels/cleanup.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { channelConfigSchema } from 'librechat-data-provider';
|
||||
import { runAsSystem, tenantStorage } from '@librechat/data-schemas';
|
||||
import type { AgentChannelMethods, ChannelRunStorage } from '@librechat/data-schemas';
|
||||
import type { ChannelConfig } from 'librechat-data-provider';
|
||||
import type { ChannelRunDependencies } from './runs';
|
||||
import { createIdleRecoveryLoop } from '~/agents/recovery';
|
||||
import { hasChannelRunAuthority } from './runs';
|
||||
|
||||
export interface ChannelCleanupDependencies {
|
||||
storage: ChannelRunStorage;
|
||||
channels: Pick<
|
||||
AgentChannelMethods,
|
||||
'getAgentChannelMetadata' | 'listDeletingAgentChannels' | 'deleteAgentChannel'
|
||||
>;
|
||||
resolvePrincipal: ChannelRunDependencies['resolvePrincipal'];
|
||||
retireForAuthorityLoss: ChannelRunDependencies['retireForAuthorityLoss'];
|
||||
isCredentialCurrent: (
|
||||
run: Parameters<ChannelRunDependencies['retireForAuthorityLoss']>[0],
|
||||
) => Promise<boolean>;
|
||||
deleteApiKey: (id: string, userId: string) => Promise<boolean>;
|
||||
getConfig: () => Promise<ChannelConfig | undefined>;
|
||||
onError: (error: unknown) => void;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Private recovery survives disabled features and missing gateway/channel rows.
|
||||
* Only metadata crosses system context; cleanup cannot enqueue or deliver. */
|
||||
export function createChannelCleanupRuntime(deps: ChannelCleanupDependencies) {
|
||||
const owner = `channel-cleanup-${randomUUID()}`;
|
||||
const now = deps.now ?? Date.now;
|
||||
let worker: ReturnType<typeof createIdleRecoveryLoop> | undefined;
|
||||
let initialization: Promise<void> | undefined;
|
||||
let stopped = false;
|
||||
let runCursor: string | undefined;
|
||||
let channelCursor: string | undefined;
|
||||
return {
|
||||
initialize(): Promise<void> {
|
||||
if (stopped) return Promise.resolve();
|
||||
if (initialization) return initialization;
|
||||
initialization = (async () => {
|
||||
const initial = channelConfigSchema.parse((await deps.getConfig()) ?? {});
|
||||
if (stopped) return;
|
||||
worker = createIdleRecoveryLoop({
|
||||
...initial.cleanup,
|
||||
onError: deps.onError,
|
||||
scan: async () => {
|
||||
if (stopped) return true;
|
||||
const config = channelConfigSchema.parse((await deps.getConfig()) ?? {});
|
||||
const pageSize = Math.min(config.cleanup.batchSize, config.maxMetadataPageSize);
|
||||
const [runs, deleting] = await runAsSystem(() =>
|
||||
Promise.all([
|
||||
deps.storage.listPendingMetadata(runCursor, pageSize),
|
||||
deps.channels.listDeletingAgentChannels(channelCursor, pageSize),
|
||||
]),
|
||||
);
|
||||
runCursor = runs.next;
|
||||
channelCursor = deleting.next;
|
||||
let idle = !runs.next && !deleting.next;
|
||||
for (const target of runs.runs) {
|
||||
if (stopped) break;
|
||||
await tenantStorage.run(
|
||||
{ tenantId: target.tenantId === '_' ? undefined : target.tenantId },
|
||||
async () => {
|
||||
try {
|
||||
const run = await deps.storage.read(
|
||||
target.tenantId,
|
||||
target.channelId,
|
||||
target.runId,
|
||||
);
|
||||
if (!run) return;
|
||||
const channel = await deps.channels.getAgentChannelMetadata(target.channelId);
|
||||
if (
|
||||
config.enabled &&
|
||||
channel &&
|
||||
(await hasChannelRunAuthority(channel, run, deps.resolvePrincipal)) &&
|
||||
(await deps.isCredentialCurrent(run))
|
||||
)
|
||||
return;
|
||||
const claimed = await deps.storage.claimAuthorityLost(
|
||||
run,
|
||||
owner,
|
||||
now(),
|
||||
config.leaseMs,
|
||||
);
|
||||
if (!claimed) return;
|
||||
idle = false;
|
||||
if ((await deps.retireForAuthorityLoss(claimed)) !== 'stopped') return;
|
||||
await deps.storage.finishAuthorityLost(
|
||||
claimed.tenantId,
|
||||
{
|
||||
runId: claimed.runId,
|
||||
owner,
|
||||
epoch: claimed.leaseEpoch,
|
||||
},
|
||||
now(),
|
||||
);
|
||||
} catch (error) {
|
||||
idle = false;
|
||||
deps.onError(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
for (const target of deleting.channels) {
|
||||
if (stopped) break;
|
||||
await tenantStorage.run({ tenantId: target.tenantId || undefined }, async () => {
|
||||
try {
|
||||
const channel = await deps.channels.getAgentChannelMetadata(target.channelId);
|
||||
if (channel?.status !== 'deleting') return;
|
||||
if (channel.apiKeyId)
|
||||
await deps.deleteApiKey(channel.apiKeyId, channel.createdBy);
|
||||
const result = await deps.channels.deleteAgentChannel(
|
||||
channel.channelId,
|
||||
channel.createdBy,
|
||||
);
|
||||
if (result.status !== 'not_found') idle = false;
|
||||
} catch (error) {
|
||||
idle = false;
|
||||
deps.onError(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
return idle;
|
||||
},
|
||||
});
|
||||
await worker.start();
|
||||
if (stopped) await worker.stop();
|
||||
})().catch(async (error: unknown) => {
|
||||
const failed = worker;
|
||||
worker = undefined;
|
||||
try {
|
||||
await failed?.stop();
|
||||
} catch {
|
||||
/* Preserve the initialization failure after attempting owned worker disposal. */
|
||||
}
|
||||
initialization = undefined;
|
||||
throw error;
|
||||
});
|
||||
return initialization;
|
||||
},
|
||||
wake(): void {
|
||||
worker?.wake();
|
||||
},
|
||||
async stop(): Promise<void> {
|
||||
stopped = true;
|
||||
try {
|
||||
await initialization;
|
||||
} catch {
|
||||
/* Failed initialization has no running worker. */
|
||||
}
|
||||
await worker?.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
209
packages/api/src/channels/credentials.ts
Normal file
209
packages/api/src/channels/credentials.ts
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
import { z } from 'zod';
|
||||
import { Constants } from 'librechat-data-provider';
|
||||
import type {
|
||||
AgentApiKeyBinding,
|
||||
AgentApiKeyListItem,
|
||||
AgentChannelRecord,
|
||||
ChannelRunRecord,
|
||||
ChannelRunStorage,
|
||||
ResponseOwner,
|
||||
ResponseStorage,
|
||||
IUser,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { Request } from 'express';
|
||||
import type { ChannelRunDependencies } from './runs';
|
||||
import { hasChannelRunAuthority } from './runs';
|
||||
|
||||
export interface ChannelResponseCredentialDependencies {
|
||||
encrypt: (value: string) => Promise<string>;
|
||||
decrypt: (value: string) => Promise<string>;
|
||||
storage: Pick<ChannelRunStorage, 'read' | 'attachResponseCredential'>;
|
||||
responses: Pick<ResponseStorage, 'read'>;
|
||||
getChannel: (id: string) => Promise<AgentChannelRecord | null>;
|
||||
resolvePrincipal: ChannelRunDependencies['resolvePrincipal'];
|
||||
createKey: (input: {
|
||||
userId: string;
|
||||
name: string;
|
||||
expiresAt: Date;
|
||||
binding: AgentApiKeyBinding;
|
||||
}) => Promise<{ id: string; key: string }>;
|
||||
getKey: (keyId: string, userId: string) => Promise<AgentApiKeyListItem | null>;
|
||||
deleteKey: (keyId: string, userId: string) => Promise<boolean>;
|
||||
lifetimeMs: () => Promise<number>;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
const requestSchema = z
|
||||
.object({
|
||||
model: z.string(),
|
||||
input: z.string(),
|
||||
background: z.literal(true),
|
||||
store: z.literal(true),
|
||||
stream: z.literal(false).optional(),
|
||||
previous_response_id: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
const bindingFor = (run: ChannelRunRecord): AgentApiKeyBinding => ({
|
||||
kind: 'channel_run',
|
||||
id: `${run.channelId}/${run.runId}`,
|
||||
});
|
||||
const previousResponse = (run: ChannelRunRecord) =>
|
||||
run.execution?.parentMessageId === Constants.NO_PARENT
|
||||
? undefined
|
||||
: run.execution?.parentMessageId;
|
||||
|
||||
/** Raw keys cross only the guarded gateway boundary; durable state retains encrypted keys and immutable grant references. */
|
||||
export function createChannelResponseCredentials(deps: ChannelResponseCredentialDependencies) {
|
||||
const now = deps.now ?? Date.now;
|
||||
async function current(
|
||||
binding: AgentApiKeyBinding,
|
||||
keyId: string,
|
||||
owner: ResponseOwner,
|
||||
agentId?: string,
|
||||
) {
|
||||
if (binding.kind !== 'channel_run' || !keyId) return null;
|
||||
const [channelId, runId, extra] = binding.id.split('/');
|
||||
if (!channelId || !runId || extra != null) return null;
|
||||
const run = await deps.storage.read(owner.tenantId || '_', channelId, runId);
|
||||
if (
|
||||
!run ||
|
||||
run.principal.userId !== owner.userId ||
|
||||
run.execution?.apiKeyId !== keyId ||
|
||||
run.execution.protocol !== 'responses' ||
|
||||
(agentId != null && run.agentId !== agentId)
|
||||
)
|
||||
return null;
|
||||
const [authorized, key] = await Promise.all([
|
||||
deps
|
||||
.getChannel(channelId)
|
||||
.then((channel) =>
|
||||
channel ? hasChannelRunAuthority(channel, run, deps.resolvePrincipal) : false,
|
||||
),
|
||||
deps.getKey(keyId, run.principal.userId),
|
||||
]);
|
||||
// A key read early must still be unexpired after the independent policy lookup finishes.
|
||||
return authorized && eligibleKey(run, key) ? run : null;
|
||||
}
|
||||
function eligibleKey(run: ChannelRunRecord, key: AgentApiKeyListItem | null): boolean {
|
||||
const execution = run.execution;
|
||||
if (!execution?.apiKeyId || execution.protocol !== 'responses' || !execution.encryptedApiKey)
|
||||
return false;
|
||||
const binding = bindingFor(run);
|
||||
return Boolean(
|
||||
key &&
|
||||
key.id === execution.apiKeyId &&
|
||||
key.binding?.kind === binding.kind &&
|
||||
key.binding.id === binding.id &&
|
||||
(key.expiresAt == null ||
|
||||
(Number.isFinite(new Date(key.expiresAt).getTime()) &&
|
||||
new Date(key.expiresAt).getTime() > now())),
|
||||
);
|
||||
}
|
||||
async function isCurrent(run: ChannelRunRecord): Promise<boolean> {
|
||||
const execution = run.execution;
|
||||
if (!execution?.apiKeyId) return true;
|
||||
if (execution.protocol !== 'responses' || !execution.encryptedApiKey) return false;
|
||||
return eligibleKey(run, await deps.getKey(execution.apiKeyId, run.principal.userId));
|
||||
}
|
||||
async function reveal(run: ChannelRunRecord): Promise<string> {
|
||||
const execution = run.execution;
|
||||
if (!execution?.apiKeyId || !execution.encryptedApiKey)
|
||||
throw new Error('Response credential unavailable');
|
||||
if (!(await isCurrent(run))) throw new Error('Response credential authority changed');
|
||||
return deps.decrypt(execution.encryptedApiKey);
|
||||
}
|
||||
return {
|
||||
isCurrent: (run: ChannelRunRecord): Promise<boolean> => isCurrent(run),
|
||||
async provision(run: ChannelRunRecord): Promise<string> {
|
||||
if (run.execution?.apiKeyId) return reveal(run);
|
||||
if (
|
||||
!run.execution ||
|
||||
run.execution.deliveryKey ||
|
||||
run.execution.enqueueAttemptedAt ||
|
||||
run.cancellationRequested ||
|
||||
!run.leaseOwner
|
||||
)
|
||||
throw new Error('Response credential admission changed');
|
||||
const lifetimeMs = await deps.lifetimeMs();
|
||||
if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs <= 0)
|
||||
throw new Error('Invalid response credential policy');
|
||||
const key = await deps.createKey({
|
||||
userId: run.principal.userId,
|
||||
name: 'Channel response run',
|
||||
expiresAt: new Date(now() + lifetimeMs),
|
||||
binding: bindingFor(run),
|
||||
});
|
||||
let saved: ChannelRunRecord;
|
||||
try {
|
||||
saved = await deps.storage.attachResponseCredential(
|
||||
run.tenantId,
|
||||
{ runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch },
|
||||
{ apiKeyId: key.id, encryptedApiKey: await deps.encrypt(key.key) },
|
||||
now(),
|
||||
);
|
||||
} catch (error) {
|
||||
const currentRun = await deps.storage.read(run.tenantId, run.channelId, run.runId);
|
||||
const winner = currentRun?.execution;
|
||||
const sameAdmission =
|
||||
currentRun &&
|
||||
winner?.apiKeyId &&
|
||||
winner.encryptedApiKey &&
|
||||
winner.protocol === 'responses' &&
|
||||
currentRun.principal.userId === run.principal.userId &&
|
||||
currentRun.agentId === run.agentId &&
|
||||
winner.requestId === run.execution.requestId &&
|
||||
currentRun.leaseOwner === run.leaseOwner &&
|
||||
currentRun.leaseEpoch === run.leaseEpoch;
|
||||
if (!sameAdmission) {
|
||||
if (winner?.apiKeyId !== key.id) await deps.deleteKey(key.id, run.principal.userId);
|
||||
throw error;
|
||||
}
|
||||
saved = currentRun;
|
||||
}
|
||||
if (saved.execution?.apiKeyId !== key.id) await deps.deleteKey(key.id, run.principal.userId);
|
||||
return reveal(saved);
|
||||
},
|
||||
async authorizeSource(
|
||||
binding: AgentApiKeyBinding,
|
||||
keyId: string,
|
||||
owner: ResponseOwner,
|
||||
agentId: string,
|
||||
): Promise<boolean> {
|
||||
return Boolean(await current(binding, keyId, owner, agentId));
|
||||
},
|
||||
async authorizeRequest(input: {
|
||||
keyId: string;
|
||||
binding: AgentApiKeyBinding;
|
||||
user: IUser;
|
||||
request: Request;
|
||||
}): Promise<boolean> {
|
||||
const owner = { userId: String(input.user._id), tenantId: input.user.tenantId };
|
||||
const run = await current(input.binding, input.keyId, owner);
|
||||
if (!run) return false;
|
||||
const pathname = input.request.originalUrl.split('?')[0];
|
||||
if (input.request.method === 'POST' && pathname === '/api/agents/v1/responses') {
|
||||
if (run.cancellationRequested) return false;
|
||||
const parsed = requestSchema.safeParse(input.request.body);
|
||||
return (
|
||||
parsed.success &&
|
||||
parsed.data.model === run.agentId &&
|
||||
parsed.data.input === run.admission.input &&
|
||||
parsed.data.previous_response_id === previousResponse(run) &&
|
||||
input.request.headers['idempotency-key'] === `channel:${run.runId}`
|
||||
);
|
||||
}
|
||||
const path = /^\/api\/agents\/v1\/responses\/(resp_[\w-]+)(\/cancel)?$/.exec(pathname);
|
||||
if (!path || (path[2] ? input.request.method !== 'POST' : input.request.method !== 'GET'))
|
||||
return false;
|
||||
if (path[2] && !run.cancellationRequested) return false;
|
||||
const response = await deps.responses.read(owner, path[1]);
|
||||
return Boolean(
|
||||
response &&
|
||||
response.agentId === run.agentId &&
|
||||
response.requestKey === `channel:${run.runId}` &&
|
||||
response.sourceKeyId === input.keyId &&
|
||||
(run.execution?.responseId == null || run.execution.responseId === response.responseId),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
19
packages/api/src/channels/errors.spec.ts
Normal file
19
packages/api/src/channels/errors.spec.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { ChannelExecutionError as CompatibilityError } from './execution';
|
||||
import { ChannelExecutionError } from './errors';
|
||||
|
||||
it.each([
|
||||
'execution_not_prepared',
|
||||
'execution_identity_changed',
|
||||
'dispatch_outcome_unknown',
|
||||
'generation_missing',
|
||||
'generation_replaced',
|
||||
'stop_unconfirmed',
|
||||
] as const)('preserves error identity and diagnostics for %s', (code) => {
|
||||
expect(CompatibilityError).toBe(ChannelExecutionError);
|
||||
const error = new ChannelExecutionError(code);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(error).toBeInstanceOf(CompatibilityError);
|
||||
expect(error.name).toBe('ChannelExecutionError');
|
||||
expect(error.message).toBe(code);
|
||||
expect(error.code).toBe(code);
|
||||
});
|
||||
14
packages/api/src/channels/errors.ts
Normal file
14
packages/api/src/channels/errors.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
export class ChannelExecutionError extends Error {
|
||||
constructor(
|
||||
readonly code:
|
||||
| 'execution_not_prepared'
|
||||
| 'execution_identity_changed'
|
||||
| 'dispatch_outcome_unknown'
|
||||
| 'generation_missing'
|
||||
| 'generation_replaced'
|
||||
| 'stop_unconfirmed',
|
||||
) {
|
||||
super(code);
|
||||
this.name = 'ChannelExecutionError';
|
||||
}
|
||||
}
|
||||
256
packages/api/src/channels/execution.spec.ts
Normal file
256
packages/api/src/channels/execution.spec.ts
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import mongoose from 'mongoose';
|
||||
import { Constants } from 'librechat-data-provider';
|
||||
import { MongoMemoryReplSet } from 'mongodb-memory-server';
|
||||
import {
|
||||
createModels,
|
||||
createMethods,
|
||||
createChannelRunStorage,
|
||||
tenantStorage,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { ChannelRunRecord } from '@librechat/data-schemas';
|
||||
import type { ChannelExecutionDependencies } from './execution';
|
||||
import { createChannelExecutionAdapter } from './execution';
|
||||
import { createChannelHandlers } from './service';
|
||||
import { createChannelRunHandlers } from './runs';
|
||||
|
||||
const scope = <T>(work: () => Promise<T>) => tenantStorage.run({ tenantId: 'tenant-a' }, work);
|
||||
const makeRun = (): ChannelRunRecord => ({
|
||||
tenantId: 'tenant-a',
|
||||
channelId: 'channel-a',
|
||||
agentId: 'agent-a',
|
||||
runId: 'channel-run-a',
|
||||
laneId: 'lane-a',
|
||||
state: 'executing',
|
||||
sequence: 1,
|
||||
receivedAt: Date.now(),
|
||||
payloadHash: 'hash-a',
|
||||
principal: {
|
||||
userId: 'user-a',
|
||||
channelRevision: 'revision-a',
|
||||
linkId: 'link-a',
|
||||
linkRevision: 'link-rev-a',
|
||||
},
|
||||
admission: {
|
||||
eventId: 'event-a',
|
||||
externalChannelId: 'C1',
|
||||
externalThreadKey: 'C1:1.001',
|
||||
externalMessageKey: 'C1:1.001',
|
||||
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
|
||||
input: 'hello',
|
||||
},
|
||||
cancellationRequested: false,
|
||||
receipts: [],
|
||||
leaseEpoch: 1,
|
||||
leaseExpiresAt: Date.now() + 60_000,
|
||||
execution: {
|
||||
requestId: 'channel-run-a',
|
||||
conversationId: 'conversation-a',
|
||||
parentMessageId: String(Constants.NO_PARENT),
|
||||
},
|
||||
});
|
||||
|
||||
describe('mounted service-token advance cannot execute an unbound run', () => {
|
||||
let mongo: MongoMemoryReplSet;
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryReplSet.create({ replSet: { count: 1 } });
|
||||
await mongoose.connect(mongo.getUri());
|
||||
createModels(mongoose);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongo?.stop();
|
||||
});
|
||||
it('rejects legacy and unprepared state behind the actual service guard without starting work', () =>
|
||||
scope(async () => {
|
||||
const fixture = makeRun();
|
||||
await createMethods(mongoose).createAgentChannel({
|
||||
channelId: fixture.channelId,
|
||||
tenantId: fixture.tenantId,
|
||||
agentId: fixture.agentId,
|
||||
createdBy: fixture.principal.userId,
|
||||
type: 'slack',
|
||||
status: 'active',
|
||||
credentialMode: 'per_user',
|
||||
revision: fixture.principal.channelRevision,
|
||||
slack: { teamId: 'T1', appId: 'A1', botUserId: 'UBOT' },
|
||||
});
|
||||
await mongoose.models.ChannelIdentity.create({
|
||||
tenantId: fixture.tenantId,
|
||||
linkId: fixture.principal.linkId,
|
||||
revision: fixture.principal.linkRevision,
|
||||
userId: fixture.principal.userId,
|
||||
state: 'active',
|
||||
ticketHash: 'ticket',
|
||||
ticketExpiresAtMs: Date.now() + 60000,
|
||||
expiresAtMs: Date.now() + 60000,
|
||||
identity: {
|
||||
tenantId: fixture.tenantId,
|
||||
channelId: fixture.channelId,
|
||||
channelRevision: fixture.principal.channelRevision,
|
||||
teamId: 'T1',
|
||||
externalUserId: 'U1',
|
||||
agentId: fixture.agentId,
|
||||
},
|
||||
});
|
||||
const storage = createChannelRunStorage(mongoose);
|
||||
await storage.admit(
|
||||
{
|
||||
tenantId: fixture.tenantId,
|
||||
channelId: fixture.channelId,
|
||||
agentId: fixture.agentId,
|
||||
principal: fixture.principal,
|
||||
admission: fixture.admission,
|
||||
},
|
||||
Date.now(),
|
||||
);
|
||||
const run = (await storage.claim(
|
||||
fixture.tenantId,
|
||||
fixture.channelId,
|
||||
'worker',
|
||||
Date.now(),
|
||||
60_000,
|
||||
))!;
|
||||
const token = 'test-service-credential-at-least-32-characters';
|
||||
const handlers = createChannelHandlers({
|
||||
db: createMethods(mongoose),
|
||||
serviceToken: () => token,
|
||||
config: async () => ({ enabled: true }),
|
||||
getAgent: async () => ({ id: fixture.agentId }),
|
||||
deleteApiKey: async () => true,
|
||||
verifySlackToken: async () => null,
|
||||
logError: jest.fn(),
|
||||
});
|
||||
const responseProjection = jest.fn();
|
||||
const adapter = createChannelExecutionAdapter({
|
||||
responseRetirement: jest.fn(),
|
||||
responseProjection,
|
||||
});
|
||||
const runs = createChannelRunHandlers({
|
||||
storage,
|
||||
guard: handlers.service,
|
||||
resolvePrincipal: async () => fixture.principal,
|
||||
advance: adapter.advance,
|
||||
retireForAuthorityLoss: adapter.retireForAuthorityLoss,
|
||||
});
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.post('/service/:channelId/runs/:runId/advance', runs.advance);
|
||||
const route = `/service/${run.channelId}/runs/${run.runId}/advance`;
|
||||
const body = { fence: { runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch } };
|
||||
await request(app).post(route).send(body).expect(401);
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await request(app)
|
||||
.post(route)
|
||||
.auth(token, { type: 'bearer' })
|
||||
.set('X-Tenant-Id', fixture.tenantId)
|
||||
.send(body)
|
||||
.expect(409, { error: 'execution_not_prepared' });
|
||||
}
|
||||
const held = await storage.read(run.tenantId, run.channelId, run.runId);
|
||||
expect(held?.execution?.enqueueAttemptedAt).toBeUndefined();
|
||||
expect(held?.execution?.deliveryKey).toBeUndefined();
|
||||
expect(held?.snapshot).toBeUndefined();
|
||||
expect(responseProjection).not.toHaveBeenCalled();
|
||||
}));
|
||||
});
|
||||
|
||||
describe('Responses-only Channels execution', () => {
|
||||
let run: ChannelRunRecord;
|
||||
let deps: ChannelExecutionDependencies;
|
||||
let adapter: ReturnType<typeof createChannelExecutionAdapter>;
|
||||
|
||||
beforeEach(() => {
|
||||
run = makeRun();
|
||||
deps = {
|
||||
responseProjection: jest.fn(async () => ({
|
||||
snapshot: { status: 'running' as const, text: '', elapsedMs: 1 },
|
||||
})),
|
||||
responseRetirement: jest.fn(async () => 'pending' as const),
|
||||
};
|
||||
adapter = createChannelExecutionAdapter(deps);
|
||||
});
|
||||
|
||||
function loseAuthority() {
|
||||
run.cancellationRequested = true;
|
||||
run.errorCode = 'channel_authority_changed';
|
||||
}
|
||||
|
||||
it.each(['unprepared', 'unbound', 'legacy_dispatched', 'responses_unbound'] as const)(
|
||||
'refuses %s advance without projecting a Response',
|
||||
async (kind) => {
|
||||
if (kind === 'unprepared') run.execution = undefined;
|
||||
if (kind === 'legacy_dispatched') run.execution!.deliveryKey = 'old-trigger';
|
||||
if (kind === 'responses_unbound') run.execution!.protocol = 'responses';
|
||||
await expect(adapter.advance(run)).rejects.toMatchObject({ code: 'execution_not_prepared' });
|
||||
expect(deps.responseProjection).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('delegates a bound Response to its canonical projection and retirement', async () => {
|
||||
run.execution = { ...run.execution!, protocol: 'responses', responseId: 'resp_saved' };
|
||||
await expect(adapter.advance(run)).resolves.toMatchObject({ snapshot: { status: 'running' } });
|
||||
expect(deps.responseProjection).toHaveBeenCalledWith(run);
|
||||
loseAuthority();
|
||||
await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('pending');
|
||||
expect(deps.responseRetirement).toHaveBeenCalledWith(run);
|
||||
});
|
||||
|
||||
it.each(['not_cancelled', 'different_reason'] as const)(
|
||||
'rejects %s cleanup before consulting Responses',
|
||||
async (kind) => {
|
||||
loseAuthority();
|
||||
if (kind === 'not_cancelled') run.cancellationRequested = false;
|
||||
if (kind === 'different_reason') run.errorCode = 'other';
|
||||
await expect(adapter.retireForAuthorityLoss(run)).rejects.toMatchObject({
|
||||
code: 'execution_identity_changed',
|
||||
});
|
||||
expect(deps.responseRetirement).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['unprepared', 'unprovisioned', 'responses_unprovisioned'] as const)(
|
||||
'retires %s authority loss without a possible provider admission',
|
||||
async (kind) => {
|
||||
loseAuthority();
|
||||
if (kind === 'unprepared') run.execution = undefined;
|
||||
if (kind === 'responses_unprovisioned') run.execution!.protocol = 'responses';
|
||||
await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('stopped');
|
||||
expect(deps.responseRetirement).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('reconciles an issued credential even when Response binding has not finished', async () => {
|
||||
loseAuthority();
|
||||
run.execution = { ...run.execution!, protocol: 'responses', apiKeyId: 'issued' };
|
||||
await expect(adapter.retireForAuthorityLoss(run)).resolves.toBe('pending');
|
||||
expect(deps.responseRetirement).toHaveBeenCalledWith(run);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'deliveryKey',
|
||||
'enqueueAttemptedAt',
|
||||
'generationCreatedAt',
|
||||
'apiKeyId',
|
||||
'responseId',
|
||||
] as const)(
|
||||
'holds unknown execution with %s instead of guessing that it stopped',
|
||||
async (field) => {
|
||||
loseAuthority();
|
||||
Object.assign(run.execution!, { [field]: field.endsWith('At') ? 100 : 'unknown' });
|
||||
await expect(adapter.retireForAuthorityLoss(run)).rejects.toMatchObject({
|
||||
code: 'execution_identity_changed',
|
||||
});
|
||||
expect(deps.responseRetirement).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('preserves canonical retirement errors for recovery to retry', async () => {
|
||||
loseAuthority();
|
||||
run.execution = { ...run.execution!, protocol: 'responses', responseId: 'resp_saved' };
|
||||
const error = new Error('storage_unavailable');
|
||||
jest.mocked(deps.responseRetirement).mockRejectedValue(error);
|
||||
await expect(adapter.retireForAuthorityLoss(run)).rejects.toBe(error);
|
||||
});
|
||||
});
|
||||
43
packages/api/src/channels/execution.ts
Normal file
43
packages/api/src/channels/execution.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
import type { ChannelRunSnapshot } from 'librechat-data-provider';
|
||||
import type { ChannelRunRecord } from '@librechat/data-schemas';
|
||||
import type { createChannelResponseProjection, createChannelResponseRetirement } from './responses';
|
||||
import { ChannelExecutionError } from './errors';
|
||||
|
||||
export { ChannelExecutionError } from './errors';
|
||||
|
||||
export interface ChannelExecutionDependencies {
|
||||
responseProjection: ReturnType<typeof createChannelResponseProjection>;
|
||||
responseRetirement: ReturnType<typeof createChannelResponseRetirement>;
|
||||
}
|
||||
|
||||
/** Execution and cancellation belong to Responses; Channels only projects delivery state. */
|
||||
export function createChannelExecutionAdapter(deps: ChannelExecutionDependencies) {
|
||||
return {
|
||||
async retireForAuthorityLoss(run: ChannelRunRecord): Promise<'stopped' | 'pending'> {
|
||||
if (!run.cancellationRequested || run.errorCode !== 'channel_authority_changed')
|
||||
throw new ChannelExecutionError('execution_identity_changed');
|
||||
if (!run.execution) return 'stopped';
|
||||
const execution = run.execution;
|
||||
// A credential permits Response admission even before its ID has been bound.
|
||||
// Only an unprovisioned run can be retired without consulting Responses.
|
||||
if (
|
||||
!execution.apiKeyId &&
|
||||
!execution.responseId &&
|
||||
execution.generationCreatedAt == null &&
|
||||
execution.deliveryKey == null &&
|
||||
execution.enqueueAttemptedAt == null
|
||||
)
|
||||
return 'stopped';
|
||||
if (execution.protocol !== 'responses')
|
||||
throw new ChannelExecutionError('execution_identity_changed');
|
||||
return deps.responseRetirement(run);
|
||||
},
|
||||
async advance(
|
||||
run: ChannelRunRecord,
|
||||
): Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }> {
|
||||
if (run.execution?.protocol !== 'responses' || !run.execution.responseId)
|
||||
throw new ChannelExecutionError('execution_not_prepared');
|
||||
return deps.responseProjection(run);
|
||||
},
|
||||
};
|
||||
}
|
||||
389
packages/api/src/channels/responses.spec.ts
Normal file
389
packages/api/src/channels/responses.spec.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryReplSet } from 'mongodb-memory-server';
|
||||
import {
|
||||
createModels,
|
||||
createMethods,
|
||||
createResponseStorage,
|
||||
tenantStorage,
|
||||
runAsSystem,
|
||||
} from '@librechat/data-schemas';
|
||||
import type {
|
||||
ChannelRunRecord,
|
||||
ResponseStorage,
|
||||
StoredResponseRecord,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { Response } from '~/agents/responses/types';
|
||||
import {
|
||||
createResponseCompletion,
|
||||
createResponseExecutionConfirmation,
|
||||
createResponseExecutionRelease,
|
||||
} from '~/agents/responses/completion';
|
||||
import {
|
||||
buildPendingBackgroundResponse,
|
||||
createBackgroundResponseService,
|
||||
} from '~/agents/responses/background';
|
||||
import { createChannelResponseProjection, createChannelResponseRetirement } from './responses';
|
||||
import { createResponseRetirement } from '~/agents/responses/retirement';
|
||||
import { GenerationJobManager } from '~/stream';
|
||||
|
||||
let mongo: MongoMemoryReplSet;
|
||||
let storage: ResponseStorage;
|
||||
let stored: StoredResponseRecord;
|
||||
let run: ChannelRunRecord;
|
||||
const owner = { userId: 'user', tenantId: 'projection-test' };
|
||||
const scoped = <T>(work: () => Promise<T>) => tenantStorage.run(owner, work);
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryReplSet.create({
|
||||
replSet: { count: 1 },
|
||||
instanceOpts: [{ args: ['--nounixsocket'] }],
|
||||
});
|
||||
await mongoose.connect(mongo.getUri(), { autoIndex: false });
|
||||
createModels(mongoose);
|
||||
storage = createResponseStorage(mongoose);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongo?.stop();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await runAsSystem(async () => {
|
||||
await mongoose.models.StoredResponse.deleteMany({});
|
||||
await mongoose.models.Conversation.deleteMany({});
|
||||
await mongoose.models.ResponseScope?.deleteMany({});
|
||||
});
|
||||
const reserved = await scoped(() =>
|
||||
storage.reserve({
|
||||
...owner,
|
||||
responseId: 'resp_saved',
|
||||
requestKey: 'channel:run',
|
||||
fingerprint: 'request',
|
||||
agentId: 'agent',
|
||||
sourceKeyId: 'key',
|
||||
requestJSON: JSON.stringify({
|
||||
model: 'agent',
|
||||
input: 'hello',
|
||||
background: true,
|
||||
store: true,
|
||||
}),
|
||||
conversationId: 'conversation',
|
||||
parentMessageId: 'root',
|
||||
}),
|
||||
);
|
||||
if (!reserved.ok) throw new Error('Reservation failed');
|
||||
stored = reserved.record;
|
||||
run = {
|
||||
tenantId: owner.tenantId,
|
||||
channelId: 'channel',
|
||||
agentId: 'agent',
|
||||
runId: 'run',
|
||||
laneId: 'lane',
|
||||
state: 'executing',
|
||||
sequence: 1,
|
||||
payloadHash: 'payload',
|
||||
receivedAt: 1000,
|
||||
principal: { userId: owner.userId, channelRevision: 'revision' },
|
||||
admission: {
|
||||
eventId: 'event',
|
||||
externalChannelId: 'C1',
|
||||
externalThreadKey: 'C1:1.0',
|
||||
externalMessageKey: 'C1:1.1',
|
||||
actor: { teamId: 'T1', appId: 'A1', externalUserId: 'U1' },
|
||||
input: 'hello',
|
||||
},
|
||||
cancellationRequested: false,
|
||||
receipts: [],
|
||||
leaseEpoch: 1,
|
||||
leaseExpiresAt: Date.now() + 10000,
|
||||
execution: {
|
||||
protocol: 'responses',
|
||||
responseId: stored.responseId,
|
||||
requestId: 'run',
|
||||
conversationId: stored.conversationId,
|
||||
parentMessageId: stored.parentMessageId,
|
||||
},
|
||||
};
|
||||
});
|
||||
function fixture() {
|
||||
let authorized = true;
|
||||
const cancel = jest.fn(async () => {});
|
||||
const service = createBackgroundResponseService({
|
||||
storage,
|
||||
recoverTerminal: createResponseCompletion({
|
||||
storage,
|
||||
cleanupCheckpoint: async () => {},
|
||||
confirmExecution: createResponseExecutionConfirmation(GenerationJobManager),
|
||||
releaseExecution: createResponseExecutionRelease(GenerationJobManager),
|
||||
}).finish,
|
||||
authorize: async () => authorized,
|
||||
enqueue: async () => {
|
||||
throw new Error('Projection must not enqueue');
|
||||
},
|
||||
resolveContinuation: async () => {
|
||||
throw new Error('Projection must not rebuild history');
|
||||
},
|
||||
observe: async () => null,
|
||||
cancel,
|
||||
});
|
||||
return {
|
||||
project: createChannelResponseProjection({
|
||||
storage,
|
||||
readResponse: service.retrieve,
|
||||
cancelResponse: service.cancel,
|
||||
now: () => 6000,
|
||||
}),
|
||||
cancel,
|
||||
revoke: () => {
|
||||
authorized = false;
|
||||
},
|
||||
};
|
||||
}
|
||||
async function publish(response: Response) {
|
||||
const result = await storage.publish(owner, stored.responseId, stored.revision, {
|
||||
status: response.status,
|
||||
responseJSON: JSON.stringify(response),
|
||||
generationCreatedAt: 1234,
|
||||
});
|
||||
if (!result.ok) throw new Error('Publication failed');
|
||||
stored = result.record;
|
||||
}
|
||||
|
||||
it('carries a nonterminal recovery warning without publishing stale actions, output or enqueueing work', async () => {
|
||||
await scoped(async () => {
|
||||
await publish({
|
||||
...buildPendingBackgroundResponse(stored),
|
||||
status: 'in_progress',
|
||||
recovery: 'execution_state_unavailable',
|
||||
});
|
||||
const f = fixture();
|
||||
const result = await f.project(run);
|
||||
expect(result.snapshot).toMatchObject({
|
||||
status: 'running',
|
||||
text: '',
|
||||
recovery: 'execution_state_unavailable',
|
||||
});
|
||||
expect(result.snapshot.pendingAction).toBeUndefined();
|
||||
expect(f.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('projects only the canonical final assistant text and actual usage', () =>
|
||||
scoped(async () => {
|
||||
const f = fixture();
|
||||
const response: Response = {
|
||||
...buildPendingBackgroundResponse(stored),
|
||||
status: 'completed',
|
||||
completed_at: 5,
|
||||
output: [
|
||||
{
|
||||
id: 'message',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
content: [{ type: 'output_text', text: '# Final answer', annotations: [], logprobs: [] }],
|
||||
},
|
||||
{
|
||||
id: 'call',
|
||||
type: 'function_call',
|
||||
call_id: 'tool',
|
||||
name: 'private_tool',
|
||||
arguments: 'PRIVATE arguments',
|
||||
status: 'completed',
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
input_tokens: 10,
|
||||
output_tokens: 2,
|
||||
total_tokens: 12,
|
||||
input_tokens_details: { cached_tokens: 3 },
|
||||
output_tokens_details: { reasoning_tokens: 1 },
|
||||
},
|
||||
};
|
||||
await publish(response);
|
||||
const result = await f.project(run);
|
||||
expect(result).toMatchObject({
|
||||
generationCreatedAt: 1234,
|
||||
snapshot: {
|
||||
responseId: 'resp_saved',
|
||||
status: 'completed',
|
||||
text: '# Final answer',
|
||||
elapsedMs: 4000,
|
||||
usage: {
|
||||
inputTokens: 10,
|
||||
outputTokens: 2,
|
||||
totalTokens: 12,
|
||||
cachedTokens: 3,
|
||||
reasoningTokens: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(result)).not.toContain('PRIVATE');
|
||||
expect(f.cancel).not.toHaveBeenCalled();
|
||||
}));
|
||||
|
||||
it('never presents intermediate output as the final Slack answer', () =>
|
||||
scoped(async () => {
|
||||
const f = fixture();
|
||||
await publish({
|
||||
...buildPendingBackgroundResponse(stored),
|
||||
status: 'in_progress',
|
||||
output: [
|
||||
{
|
||||
id: 'message',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
status: 'in_progress',
|
||||
content: [{ type: 'output_text', text: 'Still changing', annotations: [], logprobs: [] }],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect((await f.project(run)).snapshot).toMatchObject({ status: 'running', text: '' });
|
||||
}));
|
||||
|
||||
it('keeps a cancellation request running until the Responses runtime confirms settlement', () =>
|
||||
scoped(async () => {
|
||||
const f = fixture();
|
||||
await publish({ ...buildPendingBackgroundResponse(stored), status: 'in_progress' });
|
||||
run.cancellationRequested = true;
|
||||
expect((await f.project(run)).snapshot.status).toBe('running');
|
||||
expect(f.cancel).toHaveBeenCalledTimes(1);
|
||||
await publish({
|
||||
...buildPendingBackgroundResponse(stored),
|
||||
status: 'cancelled',
|
||||
completed_at: 5,
|
||||
});
|
||||
expect((await f.project(run)).snapshot.status).toBe('cancelled');
|
||||
}));
|
||||
|
||||
it('honors current source credential authority before returning output', () =>
|
||||
scoped(async () => {
|
||||
const f = fixture();
|
||||
f.revoke();
|
||||
await expect(f.project(run)).rejects.toMatchObject({ code: 'execution_identity_changed' });
|
||||
}));
|
||||
|
||||
it.each(['responseId', 'conversationId', 'parentMessageId', 'generationCreatedAt'])(
|
||||
'rejects changed %s rather than reading the latest conversation output',
|
||||
(field) =>
|
||||
scoped(async () => {
|
||||
const f = fixture();
|
||||
run.execution = {
|
||||
...run.execution!,
|
||||
[field]: field === 'generationCreatedAt' ? 5678 : 'other',
|
||||
};
|
||||
await expect(f.project(run)).rejects.toMatchObject({ code: 'execution_identity_changed' });
|
||||
}),
|
||||
);
|
||||
|
||||
it.each(['expired', 'closed'] as const)(
|
||||
'retires a %s bound Response through cleanup identity without disclosing payload',
|
||||
(state) =>
|
||||
scoped(async () => {
|
||||
run.cancellationRequested = true;
|
||||
run.errorCode = 'channel_authority_changed';
|
||||
run.execution!.apiKeyId = 'key';
|
||||
if (state === 'expired')
|
||||
await mongoose.models.StoredResponse.updateOne(
|
||||
{ responseId: stored.responseId },
|
||||
{ $set: { expiredAt: new Date(0) } },
|
||||
);
|
||||
else {
|
||||
await mongoose.models.Conversation.create({
|
||||
user: owner.userId,
|
||||
tenantId: owner.tenantId,
|
||||
conversationId: stored.conversationId,
|
||||
endpoint: 'agents',
|
||||
});
|
||||
await expect(
|
||||
createMethods(mongoose).deleteConvos(
|
||||
owner.userId,
|
||||
{ conversationId: stored.conversationId },
|
||||
{
|
||||
beforeDelete: async () => {
|
||||
throw new Error('erasure paused after scope closure');
|
||||
},
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('erasure paused after scope closure');
|
||||
}
|
||||
expect(await storage.read(owner, stored.responseId)).toBeNull();
|
||||
const jobs = {
|
||||
getJob: jest.fn(async () => undefined),
|
||||
abortJob: jest.fn(),
|
||||
finishProviderTerminalPersistence: jest.fn(),
|
||||
};
|
||||
const retire = createChannelResponseRetirement({
|
||||
storage,
|
||||
recovery: { recoverResponse: jest.fn() },
|
||||
retireResponse: createResponseRetirement({ storage, jobs }),
|
||||
});
|
||||
await expect(retire(run)).resolves.toBe('stopped');
|
||||
expect(jobs.abortJob).not.toHaveBeenCalled();
|
||||
}),
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ requestKey: 'foreign' },
|
||||
{ sourceKeyId: 'foreign' },
|
||||
{ parentMessageId: 'foreign' },
|
||||
{ agentId: 'foreign' },
|
||||
{ conversationId: 'foreign' },
|
||||
])('refuses foreign cleanup binding metadata after expiry: %j', (change) =>
|
||||
scoped(async () => {
|
||||
run.cancellationRequested = true;
|
||||
run.errorCode = 'channel_authority_changed';
|
||||
run.execution!.apiKeyId = 'key';
|
||||
await mongoose.models.StoredResponse.collection.updateOne(
|
||||
{ responseId: stored.responseId },
|
||||
{ $set: { expiredAt: new Date(0), ...change } },
|
||||
);
|
||||
const retireResponse = jest.fn(async () => 'stopped' as const);
|
||||
const retire = createChannelResponseRetirement({
|
||||
storage,
|
||||
recovery: { recoverResponse: jest.fn() },
|
||||
retireResponse,
|
||||
});
|
||||
await expect(retire(run)).rejects.toMatchObject({ code: 'execution_identity_changed' });
|
||||
expect(retireResponse).not.toHaveBeenCalled();
|
||||
}),
|
||||
);
|
||||
|
||||
it('preserves pending retirement when an expired generation has no drain proof', () =>
|
||||
scoped(async () => {
|
||||
run.cancellationRequested = true;
|
||||
run.errorCode = 'channel_authority_changed';
|
||||
run.execution!.apiKeyId = 'key';
|
||||
await mongoose.models.StoredResponse.updateOne(
|
||||
{ responseId: stored.responseId },
|
||||
{ $set: { expiredAt: new Date(0), generationCreatedAt: 1234 } },
|
||||
);
|
||||
const jobs = {
|
||||
getJob: jest.fn(async () => undefined),
|
||||
abortJob: jest.fn(),
|
||||
finishProviderTerminalPersistence: jest.fn(),
|
||||
};
|
||||
const retire = createChannelResponseRetirement({
|
||||
storage,
|
||||
recovery: { recoverResponse: jest.fn() },
|
||||
retireResponse: createResponseRetirement({ storage, jobs }),
|
||||
});
|
||||
await expect(retire(run)).resolves.toBe('pending');
|
||||
expect(jobs.abortJob).not.toHaveBeenCalled();
|
||||
}));
|
||||
|
||||
it('does not treat a cleanup metadata outage as successful retirement', () =>
|
||||
scoped(async () => {
|
||||
run.cancellationRequested = true;
|
||||
run.errorCode = 'channel_authority_changed';
|
||||
const retireResponse = jest.fn(async () => 'stopped' as const);
|
||||
const retire = createChannelResponseRetirement({
|
||||
storage: {
|
||||
readCleanupIdentity: async () => {
|
||||
throw new Error('cleanup store unavailable');
|
||||
},
|
||||
},
|
||||
recovery: { recoverResponse: jest.fn() },
|
||||
retireResponse,
|
||||
});
|
||||
await expect(retire(run)).rejects.toThrow('cleanup store unavailable');
|
||||
expect(retireResponse).not.toHaveBeenCalled();
|
||||
}));
|
||||
167
packages/api/src/channels/responses.ts
Normal file
167
packages/api/src/channels/responses.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import type {
|
||||
ChannelRunRecord,
|
||||
ChannelRunStorage,
|
||||
ResponseStorage,
|
||||
ResponseOwner,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { ChannelRunSnapshot } from 'librechat-data-provider';
|
||||
import type { BackgroundResponseResult } from '~/agents/responses/background';
|
||||
import type { Response } from '~/agents/responses/types';
|
||||
import { ChannelExecutionError } from './errors';
|
||||
|
||||
export interface ChannelResponseProjectionDependencies {
|
||||
storage: Pick<ResponseStorage, 'read'>;
|
||||
/** Reads through live source-key, owner, tenant, role and Agent authorization. */
|
||||
readResponse: (owner: ResponseOwner, responseId: string) => Promise<BackgroundResponseResult>;
|
||||
cancelResponse: (owner: ResponseOwner, responseId: string) => Promise<BackgroundResponseResult>;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Revoked channel authority can retire its exact response without recovering execution authority. */
|
||||
export function createChannelResponseRetirement(deps: {
|
||||
storage: Pick<ResponseStorage, 'readCleanupIdentity'>;
|
||||
recovery: Pick<ChannelRunStorage, 'recoverResponse'>;
|
||||
retireResponse: (owner: ResponseOwner, responseId: string) => Promise<'stopped' | 'pending'>;
|
||||
now?: () => number;
|
||||
}): (run: ChannelRunRecord) => Promise<'stopped' | 'pending'> {
|
||||
return async (run) => {
|
||||
if (
|
||||
run.execution?.protocol !== 'responses' ||
|
||||
!run.cancellationRequested ||
|
||||
run.errorCode !== 'channel_authority_changed'
|
||||
)
|
||||
throw new ChannelExecutionError('execution_identity_changed');
|
||||
if (!run.execution.responseId) {
|
||||
if (!run.leaseOwner) throw new ChannelExecutionError('execution_identity_changed');
|
||||
const recovered = await deps.recovery.recoverResponse(
|
||||
run.tenantId,
|
||||
{ runId: run.runId, owner: run.leaseOwner, epoch: run.leaseEpoch },
|
||||
(deps.now ?? Date.now)(),
|
||||
);
|
||||
if (!recovered) return 'pending';
|
||||
run = recovered;
|
||||
}
|
||||
const responseId = run.execution?.responseId;
|
||||
if (!responseId) return 'pending';
|
||||
const owner = {
|
||||
userId: run.principal.userId,
|
||||
tenantId: run.tenantId === '_' ? undefined : run.tenantId,
|
||||
};
|
||||
const stored = await deps.storage.readCleanupIdentity(owner, responseId);
|
||||
if (
|
||||
stored &&
|
||||
(stored.responseId !== responseId ||
|
||||
stored.userId !== owner.userId ||
|
||||
(stored.tenantId || undefined) !== (owner.tenantId || undefined) ||
|
||||
stored.agentId !== run.agentId ||
|
||||
stored.requestKey !== `channel:${run.runId}` ||
|
||||
stored.conversationId !== run.execution!.conversationId ||
|
||||
stored.parentMessageId !== run.execution!.parentMessageId ||
|
||||
(run.execution!.apiKeyId != null && stored.sourceKeyId !== run.execution!.apiKeyId))
|
||||
)
|
||||
throw new ChannelExecutionError('execution_identity_changed');
|
||||
// The private retirement owner resolves missing rows from cleanup/job identities;
|
||||
// absence here never proves that a provider stopped.
|
||||
return deps.retireResponse(owner, responseId);
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotStatus(response: Response): ChannelRunSnapshot['status'] {
|
||||
switch (response.status) {
|
||||
case 'completed':
|
||||
return 'completed';
|
||||
case 'cancelled':
|
||||
return 'cancelled';
|
||||
case 'failed':
|
||||
case 'incomplete':
|
||||
return 'failed';
|
||||
default:
|
||||
return response.required_action ? 'requires_action' : 'running';
|
||||
}
|
||||
}
|
||||
|
||||
/** Delivery projection only: never starts an Agent, rebuilds a transcript or infers completion. */
|
||||
export function createChannelResponseProjection(deps: ChannelResponseProjectionDependencies) {
|
||||
return async (
|
||||
run: ChannelRunRecord,
|
||||
): Promise<{ snapshot: ChannelRunSnapshot; generationCreatedAt?: number }> => {
|
||||
const execution = run.execution;
|
||||
if (execution?.protocol !== 'responses' || !execution.responseId)
|
||||
throw new ChannelExecutionError('execution_not_prepared');
|
||||
const owner = {
|
||||
userId: run.principal.userId,
|
||||
tenantId: run.tenantId === '_' ? undefined : run.tenantId,
|
||||
};
|
||||
const stored = await deps.storage.read(owner, execution.responseId);
|
||||
if (
|
||||
!stored ||
|
||||
stored.agentId !== run.agentId ||
|
||||
stored.requestKey !== `channel:${run.runId}` ||
|
||||
stored.conversationId !== execution.conversationId ||
|
||||
stored.parentMessageId !== execution.parentMessageId ||
|
||||
(execution.apiKeyId != null && stored.sourceKeyId !== execution.apiKeyId) ||
|
||||
(execution.generationCreatedAt != null &&
|
||||
stored.generationCreatedAt !== execution.generationCreatedAt)
|
||||
)
|
||||
throw new ChannelExecutionError('execution_identity_changed');
|
||||
const result = await (run.cancellationRequested ? deps.cancelResponse : deps.readResponse)(
|
||||
owner,
|
||||
execution.responseId,
|
||||
);
|
||||
if (!result.ok) throw new ChannelExecutionError('execution_identity_changed');
|
||||
const response = result.response;
|
||||
if (
|
||||
response.id !== execution.responseId ||
|
||||
response.model !== run.agentId ||
|
||||
!response.background ||
|
||||
!response.store
|
||||
)
|
||||
throw new ChannelExecutionError('execution_identity_changed');
|
||||
const text: string[] = [];
|
||||
for (const item of response.output) {
|
||||
if (item.type !== 'message' || item.role !== 'assistant') continue;
|
||||
for (const content of item.content) {
|
||||
if (content.type === 'output_text') text.push(content.text);
|
||||
}
|
||||
}
|
||||
const terminal = ['completed', 'failed', 'incomplete', 'cancelled'].includes(response.status);
|
||||
const status = snapshotStatus(response);
|
||||
const action = response.required_action;
|
||||
return {
|
||||
generationCreatedAt: stored.generationCreatedAt,
|
||||
snapshot: {
|
||||
responseId: response.id,
|
||||
status,
|
||||
...(response.recovery ? { recovery: response.recovery } : {}),
|
||||
text: terminal ? text.join('') : '',
|
||||
elapsedMs: Math.max(
|
||||
0,
|
||||
(response.completed_at == null
|
||||
? (deps.now ?? Date.now)()
|
||||
: response.completed_at * 1000) - run.receivedAt,
|
||||
),
|
||||
...(response.accounting ? { accounting: response.accounting } : {}),
|
||||
...(response.usage
|
||||
? {
|
||||
usage: {
|
||||
inputTokens: response.usage.input_tokens,
|
||||
outputTokens: response.usage.output_tokens,
|
||||
totalTokens: response.usage.total_tokens,
|
||||
cachedTokens: response.usage.input_tokens_details.cached_tokens,
|
||||
reasoningTokens: response.usage.output_tokens_details.reasoning_tokens,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(!terminal && action
|
||||
? {
|
||||
pendingAction: {
|
||||
actionId: action.action_id,
|
||||
kind: action.kind === 'question' ? ('ask_user_question' as const) : action.kind,
|
||||
expiresAt: action.expires_at * 1000,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
};
|
||||
}
|
||||
1267
packages/api/src/channels/runStorage.spec.ts
Normal file
1267
packages/api/src/channels/runStorage.spec.ts
Normal file
File diff suppressed because it is too large
Load diff
1033
packages/api/src/channels/runs.ts
Normal file
1033
packages/api/src/channels/runs.ts
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -158,12 +158,14 @@ export * from './mcp/approval';
|
|||
|
||||
export { createChannelStorageConfigReader } from './channels/config';
|
||||
export * from './channels/service';
|
||||
|
||||
export * from './channels/runs';
|
||||
export * from './channels/authority';
|
||||
|
||||
export * from './channels/execution';
|
||||
export * from './channels/responses';
|
||||
export * from './channels/credentials';
|
||||
export * from './channels/identity';
|
||||
export * from './channels/identityHttp';
|
||||
export * from './channels/artifacts';
|
||||
export * from './channels/artifactStorage';
|
||||
|
||||
export * from './channels/cleanup';
|
||||
export * from './channels/summary';
|
||||
|
|
|
|||
|
|
@ -111,6 +111,8 @@ export interface ChannelDeliveryPart {
|
|||
text: string;
|
||||
/** Provider-defined, versioned JSON presentation; immutable alongside the text fallback. */
|
||||
presentation?: { format: string; payload: string };
|
||||
/** Optional diagnostic operation; reauthorized independently of immutable answer delivery. */
|
||||
traceUrl?: string;
|
||||
/** Opaque server-authorized artifact identity; never an arbitrary URL or disk path. */
|
||||
artifactId?: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -187,6 +187,7 @@ export function assertPlan(
|
|||
(part) =>
|
||||
!part.operationId ||
|
||||
!['message', 'artifact'].includes(part.kind) ||
|
||||
(part.traceUrl != null && (part.required || part.kind !== 'message' || !part.traceUrl)) ||
|
||||
(part.kind === 'artifact' &&
|
||||
(!part.artifactId ||
|
||||
!run.snapshot?.artifacts?.some((artifact) => artifact.artifactId === part.artifactId))),
|
||||
|
|
|
|||
|
|
@ -2193,3 +2193,23 @@ it('closes and cooperatively cancels private prepared work while retaining redac
|
|||
await eraseResponseRecords(mongoose, owner, ['conversation-one']);
|
||||
expect(await mongoose.models.StoredResponse.countDocuments({})).toBe(0);
|
||||
}));
|
||||
|
||||
it.each(['live', 'expired', 'closed'] as const)(
|
||||
'returns immutable cleanup binding metadata without payload for %s rows',
|
||||
(state) =>
|
||||
scoped(async () => {
|
||||
await storage.reserve(reservation({ sourceKeyId: 'run-key' }));
|
||||
if (state === 'expired')
|
||||
await createResponseRetention(mongoose)(owner, 'conversation-one', new Date(0));
|
||||
if (state === 'closed') await closeResponseScopes(mongoose, owner, ['conversation-one']);
|
||||
const identity = await storage.readCleanupIdentity(owner, 'resp_one');
|
||||
expect(identity).toMatchObject({
|
||||
requestKey: 'request-one',
|
||||
sourceKeyId: 'run-key',
|
||||
parentMessageId: 'root',
|
||||
});
|
||||
expect(identity).not.toHaveProperty('requestJSON');
|
||||
expect(identity).not.toHaveProperty('responseJSON');
|
||||
expect(JSON.stringify(identity)).not.toContain('Hello');
|
||||
}),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -52,7 +52,8 @@ export type ResponseMutationResult =
|
|||
| { ok: true; record: StoredResponseRecord }
|
||||
| { ok: false; code: 'response_conflict' };
|
||||
|
||||
/** Internal drain identity only; expired requests, answers and decisions are never returned. */
|
||||
/** Internal drain identity only; expired requests, answers and decisions are never returned.
|
||||
* Stored rows include immutable request binding; older retained job identities may omit it. */
|
||||
export type ResponseCleanupIdentity = Pick<
|
||||
StoredResponseRecord,
|
||||
| 'userId'
|
||||
|
|
@ -62,13 +63,14 @@ export type ResponseCleanupIdentity = Pick<
|
|||
| 'conversationId'
|
||||
| 'generationCreatedAt'
|
||||
| 'expiredAt'
|
||||
> & {
|
||||
/** Cleanup authority only; never permission to read output or execute a provider. */
|
||||
retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired';
|
||||
checkpointNamespace?: string;
|
||||
checkpointConfigJSON?: string;
|
||||
requiresCheckpointCleanup?: boolean;
|
||||
};
|
||||
> &
|
||||
Partial<Pick<StoredResponseRecord, 'requestKey' | 'sourceKeyId' | 'parentMessageId'>> & {
|
||||
/** Cleanup authority only; never permission to read output or execute a provider. */
|
||||
retirement?: 'owner_deleted' | 'conversation_deleted' | 'expired';
|
||||
checkpointNamespace?: string;
|
||||
checkpointConfigJSON?: string;
|
||||
requiresCheckpointCleanup?: boolean;
|
||||
};
|
||||
|
||||
export interface ResponseResumeReservation
|
||||
extends Omit<ResponseResumeIntent, 'state' | 'reservedAt' | 'admittedAt'> {
|
||||
|
|
@ -851,7 +853,7 @@ export function createResponseStorage(
|
|||
const record = await model
|
||||
.findOne({ ...ownerScope(owner), responseId })
|
||||
.select(
|
||||
'userId tenantId responseId agentId conversationId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON preparedTerminal.checkpointNamespace preparedTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId',
|
||||
'userId tenantId responseId agentId conversationId requestKey sourceKeyId parentMessageId generationCreatedAt expiredAt pendingTerminal.checkpointNamespace pendingTerminal.checkpointConfigJSON preparedTerminal.checkpointNamespace preparedTerminal.checkpointConfigJSON review.checkpointConfigJSON review.providerExecutionId',
|
||||
)
|
||||
.lean();
|
||||
// Retained data comes only from an internal job/receipt, never request input.
|
||||
|
|
@ -928,6 +930,9 @@ export function createResponseStorage(
|
|||
responseId: id,
|
||||
agentId,
|
||||
conversationId,
|
||||
...(source.requestKey == null ? {} : { requestKey: source.requestKey }),
|
||||
...(source.sourceKeyId == null ? {} : { sourceKeyId: source.sourceKeyId }),
|
||||
...(source.parentMessageId == null ? {} : { parentMessageId: source.parentMessageId }),
|
||||
generationCreatedAt:
|
||||
generationCreatedAt ?? (retirement ? matchingRetained?.generationCreatedAt : undefined),
|
||||
expiredAt,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue