mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 22:43:38 +00:00
🏚️ feat: Recover Coding Chats With Missing Workspaces (#16273)
* 🏚️ feat: Recover Coding Chats With Missing Workspaces
* fix: Revalidate Workspace Recovery and Preserve V1 Usage
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
f13b0eaef4
commit
02492bdb6c
18 changed files with 680 additions and 97 deletions
|
|
@ -319,6 +319,97 @@ describe('getCodeBridgeWorkerStatus', () => {
|
|||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('bypasses cached status without falling back on failure or changing normal polling', async () => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const offline = () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
workerId: 'personal-vm',
|
||||
online: false,
|
||||
ready: false,
|
||||
}),
|
||||
);
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockImplementationOnce(async () => offline())
|
||||
.mockRejectedValueOnce(new Error('offline'))
|
||||
.mockImplementation(async () => offline());
|
||||
const poll = createCodeBridgeStatusPoller({ fetchImpl, maxConcurrent: 1 });
|
||||
const params = {
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
token: 'administrator-token',
|
||||
workerId: 'personal-vm',
|
||||
};
|
||||
await expect(poll(params)).resolves.toEqual({ status: 'offline' });
|
||||
await expect(poll({ ...params, bypassCache: true })).rejects.toEqual(
|
||||
expect.objectContaining({ reason: 'failed' }),
|
||||
);
|
||||
await expect(poll({ ...params, bypassCache: true })).resolves.toEqual({ status: 'offline' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
await expect(poll(params)).resolves.toEqual({ status: 'offline' });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
test('never joins an older in-flight poll and shares its upstream concurrency limit', async () => {
|
||||
const responses: Array<(response: Response) => void> = [];
|
||||
const fetchImpl = jest.fn(() => new Promise<Response>((resolve) => responses.push(resolve)));
|
||||
const poll = createCodeBridgeStatusPoller({ fetchImpl, maxConcurrent: 2 });
|
||||
const params = {
|
||||
baseURL: 'https://code.example.com/v1',
|
||||
token: 'administrator-token',
|
||||
workerId: 'personal-vm',
|
||||
};
|
||||
const previous = poll(params);
|
||||
const current = poll({ ...params, bypassCache: true });
|
||||
expect(current).not.toBe(previous);
|
||||
expect(poll(params)).toBe(previous);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
await expect(poll({ ...params, bypassCache: true })).rejects.toEqual(
|
||||
expect.objectContaining({ reason: 'busy' }),
|
||||
);
|
||||
await expect(poll({ ...params, workerId: 'second-vm' })).rejects.toEqual(
|
||||
expect.objectContaining({ reason: 'busy' }),
|
||||
);
|
||||
responses[1](
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
workerId: 'personal-vm',
|
||||
online: true,
|
||||
ready: true,
|
||||
leaseExpiresInMs: 50_000,
|
||||
capabilities: { sandboxProfile: 'native-srt', runtimes: ['bash'] },
|
||||
}),
|
||||
),
|
||||
);
|
||||
await expect(current).resolves.toEqual(expect.objectContaining({ status: 'ready' }));
|
||||
responses[0](
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
workerId: 'personal-vm',
|
||||
online: false,
|
||||
ready: false,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await expect(previous).resolves.toEqual({ status: 'offline' });
|
||||
const next = poll({ ...params, bypassCache: true });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||
responses[2](
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
workerId: 'personal-vm',
|
||||
online: false,
|
||||
ready: false,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await next;
|
||||
});
|
||||
|
||||
test('does not coalesce status requests across credential rotations', async () => {
|
||||
const fetchImpl = jest.fn().mockImplementation((_input, init) =>
|
||||
Promise.resolve(
|
||||
|
|
|
|||
|
|
@ -91,6 +91,8 @@ export function createCodeBridgeStatusPoller({
|
|||
baseURL: string;
|
||||
token: string;
|
||||
workerId: string;
|
||||
/** Force a new upstream request, bypassing cached results and older in-flight polls. */
|
||||
bypassCache?: boolean;
|
||||
}) => Promise<CodeBridgeWorkerStatus> {
|
||||
const requests = new Map<
|
||||
string,
|
||||
|
|
@ -98,6 +100,14 @@ export function createCodeBridgeStatusPoller({
|
|||
>();
|
||||
let active = 0;
|
||||
return (params) => {
|
||||
if (params.bypassCache) {
|
||||
if (active >= maxConcurrent) return Promise.reject(new CodeBridgeStatusError('busy'));
|
||||
active += 1;
|
||||
// Mutation validation needs its own observation, but still shares the polling capacity.
|
||||
return getCodeBridgeWorkerStatus({ ...params, fetchImpl }).finally(() => {
|
||||
active -= 1;
|
||||
});
|
||||
}
|
||||
const credentialId = createHash('sha256').update(params.token).digest('base64url');
|
||||
const normalizedBaseURL = params.baseURL.trim().replace(/\/+$/, '');
|
||||
const key = `${normalizedBaseURL}\u0000${params.workerId}\u0000${credentialId}`;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
mergeAccessibleCodeEnvironments,
|
||||
resolveCodeEnvironmentDecisionVersion,
|
||||
resolveCodeEnvironmentMoveVersion,
|
||||
resolveCodeEnvironmentMoveCapabilities,
|
||||
} from './config';
|
||||
|
||||
describe('resolveCodeEnvironmentDecisionVersion', () => {
|
||||
|
|
@ -32,18 +33,25 @@ describe('resolveCodeEnvironmentMoveVersion', () => {
|
|||
|
||||
it('advertises moves only where the effective policy enables them', () => {
|
||||
expect(resolveCodeEnvironmentMoveVersion(withMoves({ enabled: true }))).toBe(1);
|
||||
expect(resolveCodeEnvironmentMoveCapabilities(withMoves({ enabled: true }))).toEqual({
|
||||
codeEnvironmentMoveVersion: 1,
|
||||
codeWorkspaceRecoveryVersion: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([undefined, {}, { enabled: false }])(
|
||||
'keeps sealed decisions immovable by default: %j',
|
||||
(conversationMoves) => {
|
||||
expect(resolveCodeEnvironmentMoveVersion(withMoves(conversationMoves))).toBeUndefined();
|
||||
expect(resolveCodeEnvironmentMoveCapabilities(withMoves(conversationMoves))).toEqual({});
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps moves off without any stateful code configuration', () => {
|
||||
expect(resolveCodeEnvironmentMoveVersion({} as AppConfig)).toBeUndefined();
|
||||
expect(resolveCodeEnvironmentMoveVersion(undefined)).toBeUndefined();
|
||||
expect(resolveCodeEnvironmentMoveCapabilities({} as AppConfig)).toEqual({});
|
||||
expect(resolveCodeEnvironmentMoveCapabilities(undefined)).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@ import { logger } from '@librechat/data-schemas';
|
|||
import {
|
||||
CODE_ENVIRONMENT_DECISION_VERSION,
|
||||
CODE_ENVIRONMENT_MOVE_VERSION,
|
||||
CODE_WORKSPACE_RECOVERY_VERSION,
|
||||
EModelEndpoint,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TStartupConfig } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type {
|
||||
AccessibleCodeEnvironmentConfiguration,
|
||||
|
|
@ -44,6 +46,18 @@ export function resolveCodeEnvironmentMoveVersion(
|
|||
: undefined;
|
||||
}
|
||||
|
||||
/** Advertises recovery separately so already-open V1 clients retain ordinary environment moves. */
|
||||
export function resolveCodeEnvironmentMoveCapabilities(
|
||||
appConfig?: Pick<AppConfig, 'endpoints'> | null,
|
||||
): Pick<TStartupConfig, 'codeEnvironmentMoveVersion' | 'codeWorkspaceRecoveryVersion'> {
|
||||
const codeEnvironmentMoveVersion = resolveCodeEnvironmentMoveVersion(appConfig);
|
||||
if (codeEnvironmentMoveVersion == null) return {};
|
||||
return {
|
||||
codeEnvironmentMoveVersion,
|
||||
codeWorkspaceRecoveryVersion: CODE_WORKSPACE_RECOVERY_VERSION,
|
||||
};
|
||||
}
|
||||
|
||||
/** Enables the implicit managed route only after the versioned rollout is complete. */
|
||||
export function isImplicitStatefulCodeRouteAvailable(
|
||||
configuredVersion?: string,
|
||||
|
|
|
|||
|
|
@ -190,21 +190,15 @@ describe('resolveConversationCodeEnvironmentMove', () => {
|
|||
).toEqual({ codeWorkspaces: [vm] });
|
||||
});
|
||||
|
||||
it('never switches the workspace of an environment the decision already covers', () => {
|
||||
expect(() =>
|
||||
it('accepts an explicit workspace replacement for the caller to verify against live status', () => {
|
||||
const replacement = { environmentId: 'mac', workspaceId: 'canary' };
|
||||
expect(
|
||||
resolveConversationCodeEnvironmentMove({
|
||||
conversation: sealedOn(mac),
|
||||
from: [mac],
|
||||
to: [{ environmentId: 'mac', workspaceId: 'canary' }],
|
||||
conversation: sealedOn(mac, vm),
|
||||
from: [mac, vm],
|
||||
to: [replacement, vm],
|
||||
}),
|
||||
).toThrow(locked);
|
||||
expect(() =>
|
||||
resolveConversationCodeEnvironmentMove({
|
||||
conversation: sealedOn(mac),
|
||||
from: [mac],
|
||||
to: [{ environmentId: 'mac', workspaceId: 'canary' }, vm],
|
||||
}),
|
||||
).toThrow(locked);
|
||||
).toEqual({ codeWorkspaces: [replacement, vm] });
|
||||
});
|
||||
|
||||
it('drops an environment the agents stopped using without adding one', () => {
|
||||
|
|
|
|||
|
|
@ -124,11 +124,10 @@ export interface ConversationCodeEnvironmentMove {
|
|||
}
|
||||
|
||||
/**
|
||||
* Validates an owner's explicit move of a sealed attached decision onto the environments its
|
||||
* agents now use. A move may drop environments the agents stopped using and add ones they now use,
|
||||
* but never changes the workspace of an environment the decision already covers and never upgrades
|
||||
* a conversation that continues without an attached environment. `from` must repeat the persisted selections, so a client acting
|
||||
* on a stale view of the conversation cannot replace a decision it has not seen.
|
||||
* Validates the shape and expected decision of an owner's explicit move. The caller must verify
|
||||
* live registration of every target and, for same-environment replacements, absence of the old
|
||||
* workspace before persisting. `from` must repeat the persisted selections, so a stale client
|
||||
* cannot replace a decision it has not seen. A chat without an attached decision stays sealed.
|
||||
*/
|
||||
export function resolveConversationCodeEnvironmentMove({
|
||||
conversation,
|
||||
|
|
@ -149,19 +148,7 @@ export function resolveConversationCodeEnvironmentMove({
|
|||
if (!isCodeWorkspaceSelections(to) || to.length === 0) {
|
||||
throw new CodeWorkspaceSelectionError('invalid');
|
||||
}
|
||||
const sealed = new Map(
|
||||
persisted.codeWorkspaces.map(({ environmentId, workspaceId }) => [environmentId, workspaceId]),
|
||||
);
|
||||
let adds = false;
|
||||
for (const selection of to) {
|
||||
const sealedWorkspaceId = sealed.get(selection.environmentId);
|
||||
if (sealedWorkspaceId == null) {
|
||||
adds = true;
|
||||
} else if (sealedWorkspaceId !== selection.workspaceId) {
|
||||
throw new CodeWorkspaceSelectionError('locked');
|
||||
}
|
||||
}
|
||||
if (!adds && to.length === sealed.size) {
|
||||
if (sameSelections(to, persisted.codeWorkspaces)) {
|
||||
throw new CodeWorkspaceSelectionError('locked');
|
||||
}
|
||||
return { codeWorkspaces: canonicalSelections(to) };
|
||||
|
|
|
|||
|
|
@ -1640,8 +1640,20 @@ describe('moving a sealed conversation code-environment decision', () => {
|
|||
);
|
||||
return res;
|
||||
};
|
||||
const status = async () => {
|
||||
const res = response();
|
||||
await handlers.status(
|
||||
{
|
||||
user: { id: userId, role: 'USER' },
|
||||
params: { environmentId: vm.environmentId },
|
||||
} as never,
|
||||
res as never,
|
||||
);
|
||||
return res;
|
||||
};
|
||||
return {
|
||||
move,
|
||||
status,
|
||||
conversations,
|
||||
getConversation,
|
||||
listConversationRuns,
|
||||
|
|
@ -1669,6 +1681,120 @@ describe('moving a sealed conversation code-environment decision', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test.each(['attached', undefined] as const)(
|
||||
'recovers a missing workspace on the same machine with mode %s and keeps the new decision sealed',
|
||||
async (codeEnvironmentMode) => {
|
||||
const missing = { ...vm, workspaceId: 'deleted-project' };
|
||||
const { move, conversations, fetchImpl } = setup({
|
||||
stored: {
|
||||
conversationId: 'conversation-1',
|
||||
codeEnvironmentMode,
|
||||
codeWorkspaces: [missing],
|
||||
},
|
||||
});
|
||||
|
||||
const res = await move({ from: [missing], to: [vm] });
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(conversations.get('conversation-1')).toEqual({
|
||||
conversationId: 'conversation-1',
|
||||
codeEnvironmentMode: 'attached',
|
||||
codeWorkspaces: [vm],
|
||||
});
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
expect((await move({ from: [missing], to: [vm] })).statusCode).toBe(409);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
test.each([
|
||||
{
|
||||
name: 'old workspace restored',
|
||||
workspaces: ['deleted-project', 'project-a'],
|
||||
ready: true,
|
||||
reason: 'locked',
|
||||
},
|
||||
{ name: 'replacement removed', workspaces: ['unrelated'], ready: true, reason: 'missing' },
|
||||
{
|
||||
name: 'worker unavailable',
|
||||
workspaces: ['project-a'],
|
||||
ready: false,
|
||||
reason: 'worker_unavailable',
|
||||
},
|
||||
{ name: 'workspace still missing', workspaces: ['project-a'], ready: true, reason: undefined },
|
||||
])(
|
||||
'revalidates recovery after a cached status poll: $name',
|
||||
async ({ workspaces, ready, reason }) => {
|
||||
jest.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
const old = { ...vm, workspaceId: 'deleted-project' };
|
||||
const fetchImpl = jest
|
||||
.fn()
|
||||
.mockImplementationOnce(async () => workerStatusResponse())
|
||||
.mockImplementation(async () =>
|
||||
workerStatusResponse({ workspaces: workspaces.map((id) => ({ id })), ready }),
|
||||
);
|
||||
const { move, status, conversations, replaceDecision } = setup({
|
||||
stored: { conversationId: 'conversation-1', codeWorkspaces: [old] },
|
||||
fetchImpl,
|
||||
});
|
||||
expect((await status()).statusCode).toBe(200);
|
||||
expect((await status()).statusCode).toBe(200);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
|
||||
const res = await move({ from: [old], to: [vm] });
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
expect(res.statusCode).toBe(reason == null ? 200 : 409);
|
||||
if (reason != null) {
|
||||
expect(res.body).toEqual(expect.objectContaining({ reason }));
|
||||
expect(replaceDecision).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(conversations.get('conversation-1')?.codeWorkspaces).toEqual(
|
||||
reason == null ? [vm] : [old],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test('refuses replacement if the old workspace is still registered or was restored', async () => {
|
||||
const old = { ...vm, workspaceId: 'old-project' };
|
||||
const { move, conversations, replaceDecision, fetchImpl } = setup({
|
||||
stored: { conversationId: 'conversation-1', codeWorkspaces: [old] },
|
||||
fetchImpl: jest.fn(async () =>
|
||||
workerStatusResponse({ workspaces: [{ id: old.workspaceId }, { id: vm.workspaceId }] }),
|
||||
),
|
||||
});
|
||||
|
||||
const res = await move({ from: [old], to: [vm] });
|
||||
|
||||
expect(res.statusCode).toBe(409);
|
||||
expect(res.body).toEqual(expect.objectContaining({ reason: 'locked' }));
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
expect(replaceDecision).not.toHaveBeenCalled();
|
||||
expect(conversations.get('conversation-1')?.codeWorkspaces).toEqual([old]);
|
||||
});
|
||||
|
||||
test.each([
|
||||
{ ready: false, workspaces: [{ id: vm.workspaceId }], reason: 'worker_unavailable' },
|
||||
{ ready: true, workspaces: [{ id: 'unrelated' }], reason: 'missing' },
|
||||
{ ready: true, workspaces: [], reason: 'worker_unavailable' },
|
||||
])(
|
||||
'leaves a missing decision untouched when recovery fails: $reason',
|
||||
async ({ reason, ...status }) => {
|
||||
const old = { ...vm, workspaceId: 'deleted-project' };
|
||||
const { move, conversations, replaceDecision } = setup({
|
||||
stored: { conversationId: 'conversation-1', codeWorkspaces: [old] },
|
||||
fetchImpl: jest.fn(async () => workerStatusResponse(status)),
|
||||
});
|
||||
|
||||
const res = await move({ from: [old], to: [vm] });
|
||||
|
||||
expect(res.statusCode).toBe(409);
|
||||
expect(res.body).toEqual(expect.objectContaining({ reason }));
|
||||
expect(replaceDecision).not.toHaveBeenCalled();
|
||||
expect(conversations.get('conversation-1')?.codeWorkspaces).toEqual([old]);
|
||||
},
|
||||
);
|
||||
|
||||
test('moves a legacy decision that only stored its selections', async () => {
|
||||
const { move, conversations } = setup({
|
||||
stored: { conversationId: 'conversation-1', codeWorkspaces: [mac] },
|
||||
|
|
@ -1805,9 +1931,9 @@ describe('moving a sealed conversation code-environment decision', () => {
|
|||
polls: 0,
|
||||
},
|
||||
{
|
||||
name: 'a workspace switch inside a sealed environment',
|
||||
name: 'a replacement on an environment the caller cannot access',
|
||||
body: { from: [mac], to: [{ environmentId: 'mac', workspaceId: 'canary' }] },
|
||||
reason: 'locked',
|
||||
reason: 'invalid',
|
||||
polls: 0,
|
||||
},
|
||||
])('rejects $name without polling any worker', async ({ body, reason, polls }) => {
|
||||
|
|
|
|||
|
|
@ -325,6 +325,7 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
|
|||
async function assertWorkspaceRegistered(
|
||||
policy: WorkerPolicy,
|
||||
selection: CodeWorkspaceSelection,
|
||||
previousWorkspaceId?: string,
|
||||
): Promise<void> {
|
||||
const target = selectWorkerTarget(policy, selection.environmentId);
|
||||
if (target == null) {
|
||||
|
|
@ -340,6 +341,7 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
|
|||
baseURL: target.controlPlane.baseURL,
|
||||
token,
|
||||
workerId: target.workerId,
|
||||
bypassCache: true,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CodeBridgeStatusError) {
|
||||
|
|
@ -356,6 +358,13 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
|
|||
if (!current.workspaces.some(({ id }) => id === selection.workspaceId)) {
|
||||
throw new CodeWorkspaceSelectionError('missing');
|
||||
}
|
||||
if (
|
||||
previousWorkspaceId != null &&
|
||||
previousWorkspaceId !== selection.workspaceId &&
|
||||
current.workspaces.some(({ id }) => id === previousWorkspaceId)
|
||||
) {
|
||||
throw new CodeWorkspaceSelectionError('locked');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -421,7 +430,15 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
|
|||
}
|
||||
try {
|
||||
await Promise.all(
|
||||
move.codeWorkspaces.map((selection) => assertWorkspaceRegistered(policy, selection)),
|
||||
move.codeWorkspaces.map((selection) =>
|
||||
assertWorkspaceRegistered(
|
||||
policy,
|
||||
selection,
|
||||
conversation.codeWorkspaces?.find(
|
||||
({ environmentId }) => environmentId === selection.environmentId,
|
||||
)?.workspaceId,
|
||||
),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof CodeWorkspaceSelectionError) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue