🏚️ feat: Recover Coding Chats With Missing Workspaces (#16273)

* 🏚️ feat: Recover Coding Chats With Missing Workspaces

* fix: Revalidate Workspace Recovery and Preserve V1 Usage

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-09-24 06:03:23 -04:00 • committed by GitHub
parent f13b0eaef4
commit 02492bdb6c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 680 additions and 97 deletions

View file

@ -319,6 +319,97 @@ describe('getCodeBridgeWorkerStatus', () => {
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
test('bypasses cached status without falling back on failure or changing normal polling', async () => {
jest.spyOn(Date, 'now').mockReturnValue(1_000);
const offline = () =>
new Response(
JSON.stringify({
protocolVersion: 1,
workerId: 'personal-vm',
online: false,
ready: false,
}),
);
const fetchImpl = jest
.fn()
.mockImplementationOnce(async () => offline())
.mockRejectedValueOnce(new Error('offline'))
.mockImplementation(async () => offline());
const poll = createCodeBridgeStatusPoller({ fetchImpl, maxConcurrent: 1 });
const params = {
baseURL: 'https://code.example.com/v1',
token: 'administrator-token',
workerId: 'personal-vm',
};
await expect(poll(params)).resolves.toEqual({ status: 'offline' });
await expect(poll({ ...params, bypassCache: true })).rejects.toEqual(
expect.objectContaining({ reason: 'failed' }),
);
await expect(poll({ ...params, bypassCache: true })).resolves.toEqual({ status: 'offline' });
expect(fetchImpl).toHaveBeenCalledTimes(3);
await expect(poll(params)).resolves.toEqual({ status: 'offline' });
expect(fetchImpl).toHaveBeenCalledTimes(3);
});
test('never joins an older in-flight poll and shares its upstream concurrency limit', async () => {
const responses: Array<(response: Response) => void> = [];
const fetchImpl = jest.fn(() => new Promise<Response>((resolve) => responses.push(resolve)));
const poll = createCodeBridgeStatusPoller({ fetchImpl, maxConcurrent: 2 });
const params = {
baseURL: 'https://code.example.com/v1',
token: 'administrator-token',
workerId: 'personal-vm',
};
const previous = poll(params);
const current = poll({ ...params, bypassCache: true });
expect(current).not.toBe(previous);
expect(poll(params)).toBe(previous);
expect(fetchImpl).toHaveBeenCalledTimes(2);
await expect(poll({ ...params, bypassCache: true })).rejects.toEqual(
expect.objectContaining({ reason: 'busy' }),
);
await expect(poll({ ...params, workerId: 'second-vm' })).rejects.toEqual(
expect.objectContaining({ reason: 'busy' }),
);
responses[1](
new Response(
JSON.stringify({
protocolVersion: 1,
workerId: 'personal-vm',
online: true,
ready: true,
leaseExpiresInMs: 50_000,
capabilities: { sandboxProfile: 'native-srt', runtimes: ['bash'] },
}),
),
);
await expect(current).resolves.toEqual(expect.objectContaining({ status: 'ready' }));
responses[0](
new Response(
JSON.stringify({
protocolVersion: 1,
workerId: 'personal-vm',
online: false,
ready: false,
}),
),
);
await expect(previous).resolves.toEqual({ status: 'offline' });
const next = poll({ ...params, bypassCache: true });
expect(fetchImpl).toHaveBeenCalledTimes(3);
responses[2](
new Response(
JSON.stringify({
protocolVersion: 1,
workerId: 'personal-vm',
online: false,
ready: false,
}),
),
);
await next;
});
test('does not coalesce status requests across credential rotations', async () => {
const fetchImpl = jest.fn().mockImplementation((_input, init) =>
Promise.resolve(

View file

@ -91,6 +91,8 @@ export function createCodeBridgeStatusPoller({
baseURL: string;
token: string;
workerId: string;
/** Force a new upstream request, bypassing cached results and older in-flight polls. */
bypassCache?: boolean;
}) => Promise<CodeBridgeWorkerStatus> {
const requests = new Map<
string,
@ -98,6 +100,14 @@ export function createCodeBridgeStatusPoller({
>();
let active = 0;
return (params) => {
if (params.bypassCache) {
if (active >= maxConcurrent) return Promise.reject(new CodeBridgeStatusError('busy'));
active += 1;
// Mutation validation needs its own observation, but still shares the polling capacity.
return getCodeBridgeWorkerStatus({ ...params, fetchImpl }).finally(() => {
active -= 1;
});
}
const credentialId = createHash('sha256').update(params.token).digest('base64url');
const normalizedBaseURL = params.baseURL.trim().replace(/\/+$/, '');
const key = `${normalizedBaseURL}\u0000${params.workerId}\u0000${credentialId}`;

View file

@ -5,6 +5,7 @@ import {
mergeAccessibleCodeEnvironments,
resolveCodeEnvironmentDecisionVersion,
resolveCodeEnvironmentMoveVersion,
resolveCodeEnvironmentMoveCapabilities,
} from './config';
describe('resolveCodeEnvironmentDecisionVersion', () => {
@ -32,18 +33,25 @@ describe('resolveCodeEnvironmentMoveVersion', () => {
it('advertises moves only where the effective policy enables them', () => {
expect(resolveCodeEnvironmentMoveVersion(withMoves({ enabled: true }))).toBe(1);
expect(resolveCodeEnvironmentMoveCapabilities(withMoves({ enabled: true }))).toEqual({
codeEnvironmentMoveVersion: 1,
codeWorkspaceRecoveryVersion: 1,
});
});
it.each([undefined, {}, { enabled: false }])(
'keeps sealed decisions immovable by default: %j',
(conversationMoves) => {
expect(resolveCodeEnvironmentMoveVersion(withMoves(conversationMoves))).toBeUndefined();
expect(resolveCodeEnvironmentMoveCapabilities(withMoves(conversationMoves))).toEqual({});
},
);
it('keeps moves off without any stateful code configuration', () => {
expect(resolveCodeEnvironmentMoveVersion({} as AppConfig)).toBeUndefined();
expect(resolveCodeEnvironmentMoveVersion(undefined)).toBeUndefined();
expect(resolveCodeEnvironmentMoveCapabilities({} as AppConfig)).toEqual({});
expect(resolveCodeEnvironmentMoveCapabilities(undefined)).toEqual({});
});
});

View file

@ -2,8 +2,10 @@ import { logger } from '@librechat/data-schemas';
import {
CODE_ENVIRONMENT_DECISION_VERSION,
CODE_ENVIRONMENT_MOVE_VERSION,
CODE_WORKSPACE_RECOVERY_VERSION,
EModelEndpoint,
} from 'librechat-data-provider';
import type { TStartupConfig } from 'librechat-data-provider';
import type { AppConfig } from '@librechat/data-schemas';
import type {
AccessibleCodeEnvironmentConfiguration,
@ -44,6 +46,18 @@ export function resolveCodeEnvironmentMoveVersion(
: undefined;
}
/** Advertises recovery separately so already-open V1 clients retain ordinary environment moves. */
export function resolveCodeEnvironmentMoveCapabilities(
appConfig?: Pick<AppConfig, 'endpoints'> | null,
): Pick<TStartupConfig, 'codeEnvironmentMoveVersion' | 'codeWorkspaceRecoveryVersion'> {
const codeEnvironmentMoveVersion = resolveCodeEnvironmentMoveVersion(appConfig);
if (codeEnvironmentMoveVersion == null) return {};
return {
codeEnvironmentMoveVersion,
codeWorkspaceRecoveryVersion: CODE_WORKSPACE_RECOVERY_VERSION,
};
}
/** Enables the implicit managed route only after the versioned rollout is complete. */
export function isImplicitStatefulCodeRouteAvailable(
configuredVersion?: string,

View file

@ -190,21 +190,15 @@ describe('resolveConversationCodeEnvironmentMove', () => {
).toEqual({ codeWorkspaces: [vm] });
});
it('never switches the workspace of an environment the decision already covers', () => {
expect(() =>
it('accepts an explicit workspace replacement for the caller to verify against live status', () => {
const replacement = { environmentId: 'mac', workspaceId: 'canary' };
expect(
resolveConversationCodeEnvironmentMove({
conversation: sealedOn(mac),
from: [mac],
to: [{ environmentId: 'mac', workspaceId: 'canary' }],
conversation: sealedOn(mac, vm),
from: [mac, vm],
to: [replacement, vm],
}),
).toThrow(locked);
expect(() =>
resolveConversationCodeEnvironmentMove({
conversation: sealedOn(mac),
from: [mac],
to: [{ environmentId: 'mac', workspaceId: 'canary' }, vm],
}),
).toThrow(locked);
).toEqual({ codeWorkspaces: [replacement, vm] });
});
it('drops an environment the agents stopped using without adding one', () => {

View file

@ -124,11 +124,10 @@ export interface ConversationCodeEnvironmentMove {
}
/**
* Validates an owner's explicit move of a sealed attached decision onto the environments its
* agents now use. A move may drop environments the agents stopped using and add ones they now use,
* but never changes the workspace of an environment the decision already covers and never upgrades
* a conversation that continues without an attached environment. `from` must repeat the persisted selections, so a client acting
* on a stale view of the conversation cannot replace a decision it has not seen.
* Validates the shape and expected decision of an owner's explicit move. The caller must verify
* live registration of every target and, for same-environment replacements, absence of the old
* workspace before persisting. `from` must repeat the persisted selections, so a stale client
* cannot replace a decision it has not seen. A chat without an attached decision stays sealed.
*/
export function resolveConversationCodeEnvironmentMove({
conversation,
@ -149,19 +148,7 @@ export function resolveConversationCodeEnvironmentMove({
if (!isCodeWorkspaceSelections(to) || to.length === 0) {
throw new CodeWorkspaceSelectionError('invalid');
}
const sealed = new Map(
persisted.codeWorkspaces.map(({ environmentId, workspaceId }) => [environmentId, workspaceId]),
);
let adds = false;
for (const selection of to) {
const sealedWorkspaceId = sealed.get(selection.environmentId);
if (sealedWorkspaceId == null) {
adds = true;
} else if (sealedWorkspaceId !== selection.workspaceId) {
throw new CodeWorkspaceSelectionError('locked');
}
}
if (!adds && to.length === sealed.size) {
if (sameSelections(to, persisted.codeWorkspaces)) {
throw new CodeWorkspaceSelectionError('locked');
}
return { codeWorkspaces: canonicalSelections(to) };

View file

@ -1640,8 +1640,20 @@ describe('moving a sealed conversation code-environment decision', () => {
);
return res;
};
const status = async () => {
const res = response();
await handlers.status(
{
user: { id: userId, role: 'USER' },
params: { environmentId: vm.environmentId },
} as never,
res as never,
);
return res;
};
return {
move,
status,
conversations,
getConversation,
listConversationRuns,
@ -1669,6 +1681,120 @@ describe('moving a sealed conversation code-environment decision', () => {
);
});
test.each(['attached', undefined] as const)(
'recovers a missing workspace on the same machine with mode %s and keeps the new decision sealed',
async (codeEnvironmentMode) => {
const missing = { ...vm, workspaceId: 'deleted-project' };
const { move, conversations, fetchImpl } = setup({
stored: {
conversationId: 'conversation-1',
codeEnvironmentMode,
codeWorkspaces: [missing],
},
});
const res = await move({ from: [missing], to: [vm] });
expect(res.statusCode).toBe(200);
expect(conversations.get('conversation-1')).toEqual({
conversationId: 'conversation-1',
codeEnvironmentMode: 'attached',
codeWorkspaces: [vm],
});
expect(fetchImpl).toHaveBeenCalledTimes(1);
expect((await move({ from: [missing], to: [vm] })).statusCode).toBe(409);
expect(fetchImpl).toHaveBeenCalledTimes(1);
},
);
test.each([
{
name: 'old workspace restored',
workspaces: ['deleted-project', 'project-a'],
ready: true,
reason: 'locked',
},
{ name: 'replacement removed', workspaces: ['unrelated'], ready: true, reason: 'missing' },
{
name: 'worker unavailable',
workspaces: ['project-a'],
ready: false,
reason: 'worker_unavailable',
},
{ name: 'workspace still missing', workspaces: ['project-a'], ready: true, reason: undefined },
])(
'revalidates recovery after a cached status poll: $name',
async ({ workspaces, ready, reason }) => {
jest.spyOn(Date, 'now').mockReturnValue(1_000);
const old = { ...vm, workspaceId: 'deleted-project' };
const fetchImpl = jest
.fn()
.mockImplementationOnce(async () => workerStatusResponse())
.mockImplementation(async () =>
workerStatusResponse({ workspaces: workspaces.map((id) => ({ id })), ready }),
);
const { move, status, conversations, replaceDecision } = setup({
stored: { conversationId: 'conversation-1', codeWorkspaces: [old] },
fetchImpl,
});
expect((await status()).statusCode).toBe(200);
expect((await status()).statusCode).toBe(200);
expect(fetchImpl).toHaveBeenCalledTimes(1);
const res = await move({ from: [old], to: [vm] });
expect(fetchImpl).toHaveBeenCalledTimes(2);
expect(res.statusCode).toBe(reason == null ? 200 : 409);
if (reason != null) {
expect(res.body).toEqual(expect.objectContaining({ reason }));
expect(replaceDecision).not.toHaveBeenCalled();
}
expect(conversations.get('conversation-1')?.codeWorkspaces).toEqual(
reason == null ? [vm] : [old],
);
},
);
test('refuses replacement if the old workspace is still registered or was restored', async () => {
const old = { ...vm, workspaceId: 'old-project' };
const { move, conversations, replaceDecision, fetchImpl } = setup({
stored: { conversationId: 'conversation-1', codeWorkspaces: [old] },
fetchImpl: jest.fn(async () =>
workerStatusResponse({ workspaces: [{ id: old.workspaceId }, { id: vm.workspaceId }] }),
),
});
const res = await move({ from: [old], to: [vm] });
expect(res.statusCode).toBe(409);
expect(res.body).toEqual(expect.objectContaining({ reason: 'locked' }));
expect(fetchImpl).toHaveBeenCalledTimes(1);
expect(replaceDecision).not.toHaveBeenCalled();
expect(conversations.get('conversation-1')?.codeWorkspaces).toEqual([old]);
});
test.each([
{ ready: false, workspaces: [{ id: vm.workspaceId }], reason: 'worker_unavailable' },
{ ready: true, workspaces: [{ id: 'unrelated' }], reason: 'missing' },
{ ready: true, workspaces: [], reason: 'worker_unavailable' },
])(
'leaves a missing decision untouched when recovery fails: $reason',
async ({ reason, ...status }) => {
const old = { ...vm, workspaceId: 'deleted-project' };
const { move, conversations, replaceDecision } = setup({
stored: { conversationId: 'conversation-1', codeWorkspaces: [old] },
fetchImpl: jest.fn(async () => workerStatusResponse(status)),
});
const res = await move({ from: [old], to: [vm] });
expect(res.statusCode).toBe(409);
expect(res.body).toEqual(expect.objectContaining({ reason }));
expect(replaceDecision).not.toHaveBeenCalled();
expect(conversations.get('conversation-1')?.codeWorkspaces).toEqual([old]);
},
);
test('moves a legacy decision that only stored its selections', async () => {
const { move, conversations } = setup({
stored: { conversationId: 'conversation-1', codeWorkspaces: [mac] },
@ -1805,9 +1931,9 @@ describe('moving a sealed conversation code-environment decision', () => {
polls: 0,
},
{
name: 'a workspace switch inside a sealed environment',
name: 'a replacement on an environment the caller cannot access',
body: { from: [mac], to: [{ environmentId: 'mac', workspaceId: 'canary' }] },
reason: 'locked',
reason: 'invalid',
polls: 0,
},
])('rejects $name without polling any worker', async ({ body, reason, polls }) => {

View file

@ -325,6 +325,7 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
async function assertWorkspaceRegistered(
policy: WorkerPolicy,
selection: CodeWorkspaceSelection,
previousWorkspaceId?: string,
): Promise<void> {
const target = selectWorkerTarget(policy, selection.environmentId);
if (target == null) {
@ -340,6 +341,7 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
baseURL: target.controlPlane.baseURL,
token,
workerId: target.workerId,
bypassCache: true,
});
} catch (error) {
if (error instanceof CodeBridgeStatusError) {
@ -356,6 +358,13 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
if (!current.workspaces.some(({ id }) => id === selection.workspaceId)) {
throw new CodeWorkspaceSelectionError('missing');
}
if (
previousWorkspaceId != null &&
previousWorkspaceId !== selection.workspaceId &&
current.workspaces.some(({ id }) => id === previousWorkspaceId)
) {
throw new CodeWorkspaceSelectionError('locked');
}
}
/**
@ -421,7 +430,15 @@ export function createCodeEnvironmentHttpHandlers(deps: CodeEnvironmentHttpDeps)
}
try {
await Promise.all(
move.codeWorkspaces.map((selection) => assertWorkspaceRegistered(policy, selection)),
move.codeWorkspaces.map((selection) =>
assertWorkspaceRegistered(
policy,
selection,
conversation.codeWorkspaces?.find(
({ environmentId }) => environmentId === selection.environmentId,
)?.workspaceId,
),
),
);
} catch (error) {
if (error instanceof CodeWorkspaceSelectionError) {