🦪 fix: Preserve Generated File Byte Snapshots (#18009)

* fix: Preserve Generated File Byte Snapshots

* fix: Select Snapshot Generations Before Resource Partitioning

* fix: Select Snapshot Destinations Before File Admission

* fix: Separate Snapshot Capture Identity and File Consumers

* fix: Separate Owned References and Literal Snapshot Mounts

* fix: Clean Up Unfinished Snapshots and Normalize Upload Arbitration

* fix: Keep Shared Snapshot Mounts Ahead of Private Receipts

* test: Keep Snapshot Priority Regressions Type-Safe

* fix: Preserve Literal Snapshot Paths During Legacy Recovery

* fix: Reconcile Legacy Receipt Paths Before Recovery

* fix: Reuse Fresh Unnamed Legacy Code References

* fix: Expire Snapshot Reservations Before Provisioning

---------

Co-authored-by: Lia <lia@librechat.ai>
Co-authored-by: lia-by-librechat[bot] <328778573+lia-by-librechat[bot]@users.noreply.github.com>
This commit is contained in:
lia-by-librechat[bot] 2026-11-10 13:45:15 -05:00 • committed by GitHub
parent 413456941c
commit c102e2d498
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 2502 additions and 582 deletions

View file

@ -427,7 +427,7 @@ describe('Code Process', () => {
expect(result.file.file_id).toBe('existing-output');
expect(result.file.messageId).toBe(baseParams.messageId);
expect(mockCommitCodeFile).toHaveBeenCalledWith(
expect.objectContaining({ messageId: 'original-message' }),
expect.objectContaining({ messageId: baseParams.messageId }),
undefined,
);
expect(mockCommitFileProvision).toHaveBeenCalledWith(
@ -646,178 +646,37 @@ describe('Code Process', () => {
});
});
describe('atomic file claim (via processCodeOutput)', () => {
it('should reuse file_id from existing record via atomic claim', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-file-id',
filename: 'test-file.txt',
usage: 2,
createdAt: '2024-01-01T00:00:00.000Z',
});
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput(baseParams);
expect(mockClaimCodeFile).toHaveBeenCalledWith({
filename: 'test-file.txt',
conversationId: 'conv-123',
file_id: 'mock-uuid-1234',
user: 'user-123',
sourceDispatchedAt: expect.any(Number),
});
expect(result.file_id).toBe('existing-file-id');
expect(result.usage).toBe(3);
expect(result.createdAt).toBe('2024-01-01T00:00:00.000Z');
});
it('should create new file when no existing file found', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'mock-uuid-1234',
user: 'user-123',
});
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput(baseParams);
expect(result.file_id).toBe('mock-uuid-1234');
expect(result.usage).toBe(1);
describe('snapshot reservation', () => {
it('reserves a fresh capture rather than claiming the mutable filename slot', async () => {
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const { file } = await processCodeOutput(baseParams);
expect(mockClaimCodeFile).toHaveBeenCalledWith(
expect.objectContaining({
filename: 'test-file.txt',
conversationId: 'conv-123',
file_id: 'mock-uuid-1234',
user: 'user-123',
snapshot: true,
}),
);
expect(file.usage).toBe(1);
expect(getRetentionExpiry).toHaveBeenCalledWith(baseParams.req);
});
it('skips the file when the claim is newer than the background run (stale-harvest guard)', async () => {
/* A newer run owns the filename slot and the (filename, conversationId)
* unique index leaves stale bytes nowhere to live — the detached
* harvest must not overwrite fresh content. */
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-file-id',
filename: 'test-file.txt',
messageId: 'newer-run-msg',
updatedAt: '2024-01-02T00:00:00.000Z',
});
it('keeps an older background capture instead of discarding its history', async () => {
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const result = await processCodeOutput({
...baseParams,
freshClaimAfter: new Date('2024-01-01T00:00:00.000Z').getTime(),
});
expect(result).toBeNull();
expect(mockCommitCodeFile).not.toHaveBeenCalled();
});
it('still reuses the claim when it predates the background run', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-file-id',
filename: 'test-file.txt',
updatedAt: '2024-01-01T00:00:00.000Z',
});
mockCommitCodeFile.mockResolvedValue(true);
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const { file: result } = await processCodeOutput({
...baseParams,
freshClaimAfter: new Date('2024-01-02T00:00:00.000Z').getTime(),
});
expect(result.file_id).toBe('existing-file-id');
const { file } = await processCodeOutput({ ...baseParams, freshClaimAfter: 100 });
expect(file.metadata.sourceDispatchedAt).toBe(100);
expect(mockCommitCodeFile).toHaveBeenCalledWith(
expect.objectContaining({ file_id: 'existing-file-id' }),
new Date('2024-01-02T00:00:00.000Z').getTime(),
expect.objectContaining({ file_id: file.file_id }),
100,
);
});
it('skips when a newer task holds an unwritten claim (insert stamp, no updatedAt yet)', async () => {
/* A newer task claimed the filename but its content write is still in
* flight: the claim-insert stamp alone must trip the guard. */
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-file-id',
filename: 'test-file.txt',
metadata: {
sourceDispatchedAt: new Date('2024-01-02T00:00:00.000Z').getTime(),
},
});
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const result = await processCodeOutput({
...baseParams,
freshClaimAfter: new Date('2024-01-01T00:00:00.000Z').getTime(),
});
expect(result).toBeNull();
expect(mockCommitCodeFile).not.toHaveBeenCalled();
});
it('omits output when the commit boundary rejects an overtaken claim', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'mock-uuid-1234',
user: 'user-123',
});
it('omits output when the storage commit is not confirmed', async () => {
mockCommitCodeFile.mockResolvedValue(false);
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const result = await processCodeOutput({
...baseParams,
freshClaimAfter: new Date('2024-01-01T00:00:00.000Z').getTime(),
});
expect(result).toBeNull();
expect(mockCommitCodeFile).toHaveBeenCalledWith(
expect.objectContaining({
file_id: 'mock-uuid-1234',
metadata: expect.objectContaining({
sourceDispatchedAt: new Date('2024-01-01T00:00:00.000Z').getTime(),
}),
}),
new Date('2024-01-01T00:00:00.000Z').getTime(),
);
});
it('commits when the conditional write matches (ownership held through the write)', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-file-id',
filename: 'test-file.txt',
updatedAt: '2024-01-01T00:00:00.000Z',
});
mockCommitCodeFile.mockResolvedValueOnce(true);
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const { file: result } = await processCodeOutput({
...baseParams,
freshClaimAfter: new Date('2024-01-02T00:00:00.000Z').getTime(),
});
expect(result.file_id).toBe('existing-file-id');
});
it('lets a newer task overwrite an OLDER task that wrote late (writer dispatch order wins)', async () => {
/* Old task (dispatched Jan 1) settled late and wrote at Jan 3;
* this task was dispatched Jan 2. Wall-clock updatedAt is newer
* than our dispatch, but the WRITER is older — overwrite. */
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-file-id',
filename: 'test-file.txt',
updatedAt: '2024-01-03T00:00:00.000Z',
metadata: {
sourceDispatchedAt: new Date('2024-01-01T00:00:00.000Z').getTime(),
},
});
mockCommitCodeFile.mockResolvedValue(true);
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const { file: result } = await processCodeOutput({
...baseParams,
freshClaimAfter: new Date('2024-01-02T00:00:00.000Z').getTime(),
});
expect(result.file_id).toBe('existing-file-id');
expect(result.metadata.sourceDispatchedAt).toBe(
new Date('2024-01-02T00:00:00.000Z').getTime(),
);
expect(await processCodeOutput({ ...baseParams, freshClaimAfter: 100 })).toBeNull();
});
});
@ -861,7 +720,7 @@ describe('Code Process', () => {
'mock-uuid-1234.png',
);
expect(result.type).toBe('image/webp');
expect(result.context).toBe(FileContext.execute_code);
expect(result.context).toBe(FileContext.execute_code_snapshot);
expect(result.filename).toBe('chart.png');
});
@ -888,32 +747,12 @@ describe('Code Process', () => {
);
});
it('should update existing image file with cache-busted filepath', async () => {
const imageParams = { ...baseParams, name: 'chart.png' };
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-img-id',
usage: 1,
createdAt: '2024-01-01T00:00:00.000Z',
});
const imageBuffer = Buffer.alloc(500);
mockAxios.mockResolvedValue({ data: imageBuffer });
convertImage.mockResolvedValue({ filepath: '/images/user-123/existing-img-id.webp' });
const { file: result } = await processCodeOutput(imageParams);
expect(convertImage).toHaveBeenCalledWith(
mockReq,
imageBuffer,
'high',
'existing-img-id.png',
);
expect(result.file_id).toBe('existing-img-id');
expect(result.usage).toBe(2);
expect(result.filepath).toMatch(/^\/images\/user-123\/existing-img-id\.webp\?v=\d+$/);
expect(logger.debug).toHaveBeenCalledWith(
expect.stringContaining('Updating existing file'),
);
it('uses the snapshot object URL without mutable cache busting', async () => {
mockAxios.mockResolvedValue({ data: Buffer.alloc(500) });
convertImage.mockResolvedValue({ filepath: '/images/user-123/mock-uuid-1234.webp' });
const { file } = await processCodeOutput({ ...baseParams, name: 'chart.png' });
expect(file.filepath).toBe('/images/user-123/mock-uuid-1234.webp');
expect(file.usage).toBe(1);
});
});
@ -1353,68 +1192,22 @@ describe('Code Process', () => {
});
});
describe('usage counter increment', () => {
it('should set usage to 1 for new files', async () => {
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput(baseParams);
expect(result.usage).toBe(1);
});
it('should increment usage for existing files', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-id',
usage: 5,
createdAt: '2024-01-01',
});
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput(baseParams);
expect(result.usage).toBe(6);
});
it('should handle existing file with undefined usage', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-id',
createdAt: '2024-01-01',
});
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput(baseParams);
expect(result.usage).toBe(1);
});
});
describe('metadata and file properties', () => {
it('should include codeEnvRef in metadata with kind: user', async () => {
it('should include codeEnvSource in metadata with kind: user', async () => {
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput(baseParams);
expect(result.metadata).toEqual({
codeEnvRef: {
codeEnvSource: {
kind: 'user',
id: 'user-123',
storage_session_id: 'session-123',
file_id: 'file-id-123',
executionProfile: 'default',
},
codeEnvRefs: {
default: {
kind: 'user',
id: 'user-123',
storage_session_id: 'session-123',
file_id: 'file-id-123',
executionProfile: 'default',
},
},
codeDestination: true,
sourceDispatchedAt: expect.any(Number),
});
});
@ -1430,10 +1223,10 @@ describe('Code Process', () => {
executionRouteKey,
});
expect(result.metadata.codeEnvRef).toEqual(
expect(result.metadata.codeEnvSource).toEqual(
expect.objectContaining({ executionProfile: 'stateful', executionRouteKey }),
);
expect(result.metadata.codeEnvRefs[executionRouteKey]).toEqual(result.metadata.codeEnvRef);
expect(result.metadata.codeEnvRefs).toBeUndefined();
});
/* Phase C lock-in: outputs are ALWAYS user-scoped, never skill-scoped.
@ -1460,14 +1253,14 @@ describe('Code Process', () => {
// Each user's output ref is keyed by their own user id. The
// `id` field tracks the requesting user, never the skill.
expect(outputA.metadata.codeEnvRef).toEqual({
expect(outputA.metadata.codeEnvSource).toEqual({
kind: 'user',
id: 'user-A',
storage_session_id: 'session-123',
file_id: 'file-id-123',
executionProfile: 'default',
});
expect(outputB.metadata.codeEnvRef).toEqual({
expect(outputB.metadata.codeEnvSource).toEqual({
kind: 'user',
id: 'user-B',
storage_session_id: 'session-123',
@ -1476,8 +1269,8 @@ describe('Code Process', () => {
});
// No skill identity leaks into the output ref under any property.
const refA = outputA.metadata.codeEnvRef;
const refB = outputB.metadata.codeEnvRef;
const refA = outputA.metadata.codeEnvSource;
const refB = outputB.metadata.codeEnvSource;
expect(refA.kind).not.toBe('skill');
expect(refB.kind).not.toBe('skill');
expect(refA).not.toHaveProperty('version');
@ -1490,7 +1283,7 @@ describe('Code Process', () => {
const { file: result } = await processCodeOutput(baseParams);
expect(result.context).toBe(FileContext.execute_code);
expect(result.context).toBe(FileContext.execute_code_snapshot);
});
it('should include toolCallId and messageId in result', async () => {
@ -1512,172 +1305,13 @@ describe('Code Process', () => {
expect(mockCommitCodeFile).toHaveBeenCalledWith(
expect.objectContaining({
file_id: 'mock-uuid-1234',
context: FileContext.execute_code,
context: FileContext.execute_code_snapshot,
}),
undefined,
);
});
});
describe('persistedMessageId (regression for cross-turn priming)', () => {
/**
* `getCodeGeneratedFiles` filters by `messageId IN <thread message ids>`
* to scope files to the current branch. If `processCodeOutput` overwrote
* the file's `messageId` with the current run's id on every update, a
* file re-touched by a later turn (e.g. a failed read attempt that
* re-shells the same filename) would lose its link to the assistant
* message that originally produced it. Subsequent turns then can't find
* it via `getCodeGeneratedFiles`, the priming chain has nothing to seed,
* and the model thinks its own prior-turn artifact disappeared.
*
* Contract:
* - On UPDATE (claimCodeFile returned an existing record): the persisted
* `messageId` is `claimed.messageId` (preserved). Falls back to the
* current run's `messageId` when the existing record predates the
* `messageId` field (legacy data).
* - On CREATE (new file): the persisted `messageId` is the current run's.
* - The runtime return value ALWAYS uses the current run's `messageId`
* via `Object.assign(file, { messageId, toolCallId })` so the artifact
* attaches to the correct tool_call in the live response.
*/
/**
* `processCodeOutput` mutates the file object after `commitCodeFile` returns
* (`Object.assign(file, { messageId, toolCallId })`) so the runtime
* caller sees the live messageId on the response. Reading
* `mockCommitCodeFile.mock.calls[0][0]` directly would therefore reflect the
* post-mutation state because JS captures by reference. To assert
* what was actually PERSISTED, snapshot the args at call time.
*/
function snapshotCommitCodeFileArgs() {
const snapshots = [];
mockCommitCodeFile.mockImplementation(async (file) => {
snapshots.push({ ...file });
return true;
});
return snapshots;
}
it('preserves the original messageId in the persisted record on UPDATE', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-id',
filename: 'sentinel.txt',
usage: 1,
createdAt: '2024-01-01T00:00:00.000Z',
messageId: 'turn-1-original-msg',
});
const persisted = snapshotCommitCodeFileArgs();
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
await processCodeOutput({
...baseParams,
name: 'sentinel.txt',
messageId: 'turn-2-current-run-msg',
});
expect(persisted[0].messageId).toBe('turn-1-original-msg');
});
it('falls back to current run messageId on UPDATE when claimed.messageId is undefined (legacy record)', async () => {
// Legacy record predates the persistedMessageId tracking.
mockClaimCodeFile.mockResolvedValue({
file_id: 'legacy-id',
filename: 'legacy.txt',
usage: 1,
createdAt: '2024-01-01T00:00:00.000Z',
// messageId intentionally absent
});
const persisted = snapshotCommitCodeFileArgs();
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
await processCodeOutput({
...baseParams,
name: 'legacy.txt',
messageId: 'turn-N-current-run-msg',
});
expect(persisted[0].messageId).toBe('turn-N-current-run-msg');
});
it('uses the current run messageId on CREATE (no claimed record)', async () => {
mockClaimCodeFile.mockResolvedValue({
file_id: 'mock-uuid-1234',
user: 'user-123',
});
const persisted = snapshotCommitCodeFileArgs();
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
await processCodeOutput({
...baseParams,
messageId: 'turn-1-create-msg',
});
expect(persisted[0].messageId).toBe('turn-1-create-msg');
});
it('returns the CURRENT run messageId in the runtime response even on UPDATE (artifact attribution)', async () => {
// The persisted DB record keeps the original messageId, but the
// returned object surfaces the live messageId so the artifact lands
// on the correct tool_call in this run's response.
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-id',
filename: 'sentinel.txt',
usage: 1,
createdAt: '2024-01-01T00:00:00.000Z',
messageId: 'turn-1-original-msg',
});
const persisted = snapshotCommitCodeFileArgs();
const smallBuffer = Buffer.alloc(100);
mockAxios.mockResolvedValue({ data: smallBuffer });
const { file: result } = await processCodeOutput({
...baseParams,
name: 'sentinel.txt',
messageId: 'turn-2-current-run-msg',
});
// DB preserves original
expect(persisted[0].messageId).toBe('turn-1-original-msg');
// Runtime return surfaces the live (current) messageId
expect(result.messageId).toBe('turn-2-current-run-msg');
});
it('preserves the original messageId on UPDATE for image files too', async () => {
// Same contract as text files; the image branch builds its own file
// record and would silently regress if the ternary diverged there.
mockClaimCodeFile.mockResolvedValue({
file_id: 'existing-img',
filename: 'chart.png',
usage: 1,
createdAt: '2024-01-01T00:00:00.000Z',
messageId: 'turn-1-image-msg',
});
const persisted = snapshotCommitCodeFileArgs();
const imageBuffer = Buffer.alloc(500);
mockAxios.mockResolvedValue({ data: imageBuffer });
convertImage.mockResolvedValue({
filepath: '/uploads/chart.webp',
bytes: 400,
});
await processCodeOutput({
...baseParams,
name: 'chart.png',
messageId: 'turn-2-current-img-msg',
});
expect(persisted[0].messageId).toBe('turn-1-image-msg');
});
});
describe('socket pool isolation', () => {
it('should pass dedicated keepAlive:false agents to axios for processCodeOutput', async () => {
const smallBuffer = Buffer.alloc(100);
@ -4799,6 +4433,63 @@ describe('Code Process', () => {
}
});
it('reuploads a displaced private receipt before mounting a shared snapshot', async () => {
setupActiveSessions();
const upload = jest.fn(async ({ filename }) => ({
storage_session_id: `fresh-${filename}`,
file_id: `fresh-${filename}`,
filename,
}));
getStrategyFunctions.mockImplementation(() => ({
getDownloadStream: jest.fn().mockResolvedValue('primary-bytes'),
handleFileUpload: upload,
}));
const privateFile = codeFile({
file_id: 'private',
filename: 'report.txt',
sandboxFilename: 'report.txt',
storage_session_id: 'private-session',
});
privateFile.type = 'text/plain';
privateFile.metadata.codeEnvRef.kind = 'agent';
privateFile.metadata.codeEnvRef.id = 'agent-id';
getFiles.mockResolvedValue([
privateFile,
{
file_id: 'shared',
user: 'user-123',
filename: 'report.txt',
type: 'text/plain',
context: FileContext.execute_code_snapshot,
source: 'local',
filepath: '/uploads/shared',
},
]);
const result = await prime({
execute_code: { file_ids: ['private'], files: [{ file_id: 'shared' }] },
});
expect(result.files).toHaveLength(2);
expect(result.files.find((file) => file.kind === 'user')).toMatchObject({
name: 'report.txt',
id: 'fresh-report.txt',
});
const privateMount = result.files.find((file) => file.kind === 'agent');
expect(privateMount).toMatchObject({
name: expect.stringMatching(/^report-.+\.txt$/),
resource_id: 'agent-id',
});
expect(privateMount.id).toBe(`fresh-${privateMount.name}`);
expect(upload).toHaveBeenCalledWith(
expect.objectContaining({
filename: privateMount.name,
stream: 'primary-bytes',
kind: 'agent',
id: 'agent-id',
}),
);
expect(result.toolContext).toContain(privateMount.name);
});
it('recovers a converted stale image even when its storage session has a fresh sibling', async () => {
const upload = jest
.fn()

View file

@ -98,6 +98,8 @@ const mockOptionalChainWithEmptyCheck = jest.fn();
const mockGetThreadData = jest.fn();
jest.mock('~/utils', () => ({
createCodeApiRateLimitBudget: (...args: unknown[]) =>
jest.requireActual('~/utils').createCodeApiRateLimitBudget(...args),
extractLibreChatParams: (...args: unknown[]) => mockExtractLibreChatParams(...args),
getModelMaxTokens: (...args: unknown[]) => mockGetModelMaxTokens(...args),
optionalChainWithEmptyCheck: (...args: unknown[]) => mockOptionalChainWithEmptyCheck(...args),
@ -139,6 +141,9 @@ jest.mock('../../middleware/modelBoundContent', () => {
});
import { initializeAgent } from '../initialize';
import { createProvisionFilesCallback } from '~/files/provision/callback';
import { createFileProvisionRecovery } from '~/files/operations';
import { selectCodeFiles } from '~/files/code/priming';
import { primeResources } from '../resources';
import { isFatalAgentInitializationError } from '../errors';
import { canAgentGraphPause } from '../hitl/admission';
@ -7202,3 +7207,383 @@ describe('initializeAgent — deferral without a loader registry', () => {
expect(result.hasDeferredTools).toBe(true);
});
});
describe('snapshot destination admission invariants', () => {
const MB = 1024 * 1024;
const filterMock = jest.requireMock('~/files') as {
filterFilesByEndpointRuntimeConfig: jest.Mock;
};
const realFilters = jest.requireActual<typeof import('~/files/filter')>('~/files/filter');
const realResources = jest.requireActual<typeof import('../resources')>('../resources');
beforeEach(() => {
jest.clearAllMocks();
filterMock.filterFilesByEndpointRuntimeConfig
.mockReset()
.mockImplementation(realFilters.filterFilesByEndpointRuntimeConfig);
jest.mocked(primeResources).mockReset().mockImplementation(realResources.primeResources);
});
afterEach(() => {
filterMock.filterFilesByEndpointRuntimeConfig.mockReset().mockReturnValue([]);
jest.mocked(primeResources).mockReset().mockResolvedValue({
attachments: [],
requestAttachments: [],
agentContextAttachments: undefined,
tool_resources: undefined,
warnings: [],
});
});
function snapshot(id: string, time: number, overrides: Partial<TFile> = {}): TFile {
return {
file_id: id,
user: 'user-1',
tenantId: 'tenant-1',
filename: 'report.txt',
filepath: `/uploads/${id}`,
source: FileSources.local,
type: 'text/plain',
bytes: 12 * MB,
object: 'file',
embedded: false,
usage: 0,
context: FileContext.execute_code_snapshot,
createdAt: new Date(time),
metadata: {
codeDestination: true,
sourceDispatchedAt: time,
codeEnvSource: {
kind: 'user',
id: 'user-1',
file_id: 'live-report',
storage_session_id: 'sandbox',
},
},
...overrides,
};
}
async function run(
files: TFile[],
deferred: TFile[] = [],
config: { fileSizeLimit?: number; supportedMimeTypes?: string[]; disabled?: boolean } = {},
filters?: FiltersConfig,
tools: string[] = [Tools.execute_code],
) {
const { agent, req, res, loadTools, db } = createMocks();
agent.tools = tools;
agent.endpoint = EModelEndpoint.openAI;
req.user!.tenantId = 'tenant-1';
req.config!.filters = filters;
req.config!.fileConfig = {
endpoints: { openAI: { totalSizeLimit: 20, fileSizeLimit: 20, ...config } },
};
mockExtractLibreChatParams.mockReturnValue({
resendFiles: true,
modelOptions: { model: agent.model },
});
const getFiles = jest.fn().mockResolvedValue([...files, ...deferred]);
const result = await initializeAgent(
{
req,
res,
agent,
loadTools,
conversationId: 'snapshot-conversation',
codeEnvAvailable: true,
endpointOption: { endpoint: EModelEndpoint.agents },
allowedProviders: new Set([Providers.OPENAI]),
},
{
...db,
getFiles,
getConvoFiles: jest
.fn()
.mockResolvedValue([...files, ...deferred].map((file) => file.file_id)),
getCodeGeneratedFiles: jest
.fn()
.mockResolvedValue(
files.filter((file) => file.context === FileContext.execute_code_snapshot),
),
getToolFilesByIds: jest
.fn()
.mockResolvedValue(
files.filter((file) => file.context !== FileContext.execute_code_snapshot),
),
getDeferredProvisionFiles: jest.fn().mockResolvedValue(deferred),
},
);
const eager = await selectCodeFiles({
files: result.tool_resources?.execute_code?.files ?? [],
routeKey: 'default',
getFileInfo: async () => ({ lastModified: new Date().toISOString() }),
});
const staged = new Map<string, string>();
const callback = createProvisionFilesCallback({
req,
agentToolContexts: new Map([['agent-1', result]]),
fileStorage: {
recovery: createFileProvisionRecovery(),
acquireFileProvision: jest.fn().mockResolvedValue(true),
releaseFileProvision: jest.fn().mockResolvedValue(true),
commitFileProvision: jest.fn().mockResolvedValue(true),
},
provisionToCodeEnv: async ({ file, sandboxFilename }) => {
const name = sandboxFilename ?? file.filename;
staged.set(name, file.file_id);
return {
referenceSet: {},
sandboxFilename: name,
refUpdate: {
file_id: file.file_id,
routeKey: 'default',
ref: {
kind: 'user',
id: 'user-1',
file_id: file.file_id,
storage_session_id: 'sandbox',
sandboxFilename: name,
},
},
};
},
provisionToVectorDB: async ({ file }) => ({
embedded: true,
fileUpdate: { file_id: file.file_id, embedded: true },
}),
updateFile: async () => undefined,
updateCodeEnvRef: async () => undefined,
addEmbeddedEntity: async () => undefined,
});
await callback(tools, 'agent-1');
expect(getFiles).toHaveBeenCalledTimes(1);
expect(getFiles).toHaveBeenCalledWith(
expect.objectContaining({ user: 'user-1', tenantId: 'tenant-1' }),
{},
{},
);
return { result, staged, eager };
}
it.each([false, true])(
'selects two 12 MiB versions before the 20 MiB budget (deferred newest %s)',
async (deferred) => {
const old = snapshot('v1', 100);
const newest = snapshot('v2', 200);
const { staged } = await run(deferred ? [old] : [old, newest], deferred ? [newest] : []);
expect(staged.get('report.txt')).toBe('v2');
expect([...staged.values()]).not.toContain('v1');
},
);
it.each(['oversized', 'mime', 'disabled'] as const)(
'never revives an older version when the newest is %s',
async (policy) => {
const overrides: Partial<TFile> = {};
const config: Parameters<typeof run>[2] = {};
if (policy === 'oversized') overrides.bytes = 25 * MB;
if (policy === 'mime') {
overrides.type = 'application/pdf';
config.supportedMimeTypes = ['text/plain'];
}
if (policy === 'disabled') config.disabled = true;
const { staged } = await run(
[snapshot('v1', 100), snapshot('v2', 200, overrides)],
[],
config,
);
expect(staged.size).toBe(0);
},
);
it('inspects the selected generation instead of reviving safe older bytes', async () => {
await expect(
run(
[
snapshot('v1', 100, { text: 'Safe old content' }),
snapshot('v2', 200, { text: 'BLOCKED' }),
],
[],
{},
{
files: {
pii: {
fields: ['extracted_text'],
starterPatterns: [],
customPatterns: [
{ id: 'snapshot-policy', label: 'Snapshot policy', regex: 'BLOCKED' },
],
},
},
},
),
).rejects.toThrow();
});
it.each([
{ tools: [Tools.execute_code, Tools.file_search] },
{ tools: [Tools.execute_code] },
{ tools: [Tools.file_search] },
])('keeps independent consumer records when tools are $tools', async ({ tools }) => {
const upload = snapshot('upload', 100, {
context: FileContext.message_attachment,
bytes: MB,
embedded: true,
metadata: { destinationChosen: true, codeDestination: true, searchDestination: true },
});
const output = snapshot('output', 200, { bytes: MB });
const { result, staged } = await run([upload, output], [], {}, undefined, tools);
expect(staged.get('report.txt')).toBe(
tools.includes(Tools.execute_code) ? 'output' : undefined,
);
const search = result.tool_resources?.file_search?.files?.map((file) => file.file_id) ?? [];
expect(search).toEqual(tools.includes(Tools.file_search) ? ['upload'] : []);
});
it.each([
{ deferred: false, tools: [Tools.execute_code, Tools.file_search] },
{ deferred: true, tools: [Tools.execute_code, Tools.file_search] },
{ deferred: false, tools: [Tools.execute_code] },
{ deferred: false, tools: [Tools.file_search] },
])(
'keeps newer search-only intent independent (deferred $deferred, tools $tools)',
async ({ deferred, tools }) => {
const output = snapshot('output', 100, { bytes: MB });
const upload = snapshot('search', 200, {
context: FileContext.message_attachment,
bytes: MB,
embedded: !deferred,
metadata: { destinationChosen: true, searchDestination: true },
});
const { result, staged } = await run(
deferred ? [output] : [output, upload],
deferred ? [upload] : [],
{},
undefined,
tools,
);
expect(staged.get('report.txt')).toBe(
tools.includes(Tools.execute_code) ? 'output' : undefined,
);
expect(result.tool_resources?.file_search?.files?.map((file) => file.file_id) ?? []).toEqual(
tools.includes(Tools.file_search) ? ['search'] : [],
);
},
);
it('retains model-bound input independently of its superseded code copy', async () => {
const input = snapshot('model-input', 100, {
context: FileContext.message_attachment,
bytes: MB,
llmDeliveryPath: 'text',
text: 'Prompt input',
metadata: { destinationChosen: true, codeDestination: true },
});
const { result, staged } = await run([input, snapshot('output', 200, { bytes: MB })]);
expect(staged.get('report.txt')).toBe('output');
expect(result.attachments?.map((file) => file?.file_id)).toContain('model-input');
});
it.each(['oversized', 'mime'] as const)(
'keeps search input but never restores its old code bytes when newest is %s',
async (policy) => {
const upload = snapshot('upload', 100, {
context: FileContext.message_attachment,
bytes: MB,
embedded: true,
metadata: { destinationChosen: true, codeDestination: true, searchDestination: true },
});
const output = snapshot(
'output',
200,
policy === 'oversized' ? { bytes: 25 * MB } : { bytes: MB, type: 'application/pdf' },
);
const { result, staged } = await run(
[upload, output],
[],
policy === 'mime' ? { supportedMimeTypes: ['text/plain'] } : {},
undefined,
[Tools.execute_code, Tools.file_search],
);
expect(staged.size).toBe(0);
expect(result.tool_resources?.file_search?.files?.map((file) => file.file_id)).toEqual([
'upload',
]);
},
);
it('charges search-needed superseded inputs without reviving them for code when the union exceeds budget', async () => {
const upload = snapshot('upload', 100, {
context: FileContext.message_attachment,
embedded: true,
metadata: { destinationChosen: true, codeDestination: true, searchDestination: true },
});
const { result, staged } = await run([upload, snapshot('output', 200)], [], {}, undefined, [
Tools.execute_code,
Tools.file_search,
]);
expect(staged.size).toBe(0);
expect(result.tool_resources?.file_search?.files?.map((file) => file.file_id)).toEqual([
'upload',
]);
});
it.each([
[FileContext.execute_code_snapshot, true, false],
[FileContext.execute_code, true, false],
[FileContext.execute_code, false, false],
[FileContext.execute_code, true, true],
[FileContext.execute_code, false, true],
] as const)(
'charges independent capture and assigned-path writes separately across repeated initialization (%s, named %s, deferred newest %s)',
async (context, named, deferred) => {
const original = snapshot('png', 100, {
context,
filename: 'nested/chart.png',
type: 'image/webp',
metadata: {
codeDestination: true,
sourceDispatchedAt: 100,
codeEnvRef: {
kind: 'user',
id: 'user-1',
storage_session_id: 'sandbox',
file_id: 'png-copy',
sandboxFilename: named ? 'nested/chart.webp' : undefined,
},
},
});
const literal = snapshot('literal', 200, {
filename: 'nested/chart.webp',
type: 'image/webp',
});
const first = await run(
deferred ? [original] : [original, literal],
deferred ? [literal] : [],
);
expect([...first.staged.values()]).toEqual(['png']);
expect(first.staged.has('nested/chart.webp')).toBe(false);
expect(first.eager.selected.map((entry) => entry.sandboxName)).not.toContain(
'nested/chart.webp',
);
const again = await run(
deferred ? [original] : [original, literal],
deferred ? [literal] : [],
);
expect(again.staged).toEqual(first.staged);
},
);
it.each([false, true])(
'normalizes an unreferenced converted upload before code admission (deferred newest %s)',
async (deferred) => {
const upload = snapshot('converted-upload', 100, {
filename: 'chart.png',
type: 'image/webp',
context: FileContext.message_attachment,
metadata: { destinationChosen: false, codeDestination: true },
});
const output = snapshot('literal', 200, { filename: 'chart.webp', type: 'image/webp' });
const { staged } = await run(
deferred ? [upload] : [upload, output],
deferred ? [output] : [],
);
expect([...staged.entries()]).toEqual([['chart.webp', 'literal']]);
},
);
it('keeps unrelated files subject to their original shared budget and order', async () => {
const { staged } = await run([
snapshot('first', 100, { filename: 'first.txt' }),
snapshot('other', 200, { filename: 'other.txt', bytes: 9 * MB }),
]);
expect([...staged.entries()]).toEqual([['first.txt', 'first']]);
});
});

View file

@ -44,6 +44,7 @@ export function isToolOwnedAttachment(file: AttachmentTelemetryFile): boolean {
| undefined;
return (
file.context === FileContext.execute_code ||
file.context === FileContext.execute_code_snapshot ||
file.embedded === true ||
metadata?.searchDestination === true ||
metadata?.codeDestination === true ||

View file

@ -125,6 +125,7 @@ import {
} from '../projects/context';
import { assertChatProjectInstructions, ChatProjectResourcesChangedError } from '../projects/turn';
import { extractAgentContent, extractSkillContent } from '../protection/adapters/submissions';
import { selectCodeFileGenerations, planCodeSnapshotMounts } from '~/files/code/destinations';
import { createConfiguredContentInspector, inspectContent } from '../protection/runtime';
import { assertAgentAttachmentLimits, isModelBoundAttachmentFile } from './attachments';
import { assertModelBoundContent } from '../middleware/modelBoundContent';
@ -147,6 +148,7 @@ import { resolveAgentCodeExecution } from '~/code/agent';
import { hasActiveFileFieldPolicy } from '~/protection';
import { applyBackgroundToolCalls } from './background';
import { admitCurrentFileBudget } from './files/budget';
import { allowsFileResource } from '~/files/consumers';
import { applyDeferredToolLoading } from './deferral';
import { applyTurnDelivery } from './files/delivery';
import { generateArtifactsPrompt } from '~/prompts';
@ -1923,13 +1925,45 @@ export async function initializeAgent(
currentFiles = requestUsageFiles.concat(toolUsageFiles);
}
if (currentFiles?.length) {
/* Before any check reads the route: endpoint filtering, model-bound limits and content
* inspection below all have to judge each file by the route this turn delivers it by. */
currentFiles = applyTurnDelivery(currentFiles, {
routing: deliveryRouting,
consumers: fileConsumers,
});
}
const excludedCodeFileIds = new Set<string>();
let codeSnapshotMounts: Map<string, string> | undefined;
if (wantsCodeFiles) {
const membership = new Map<EToolResources, ReadonlySet<string>>([
[EToolResources.execute_code, new Set(agent.tool_resources?.execute_code?.file_ids)],
[EToolResources.file_search, new Set(agent.tool_resources?.file_search?.file_ids)],
]);
const candidates = [...(currentFiles ?? []), ...deferredProvisionFiles].filter((file) =>
allowsFileResource(file, EToolResources.execute_code, membership),
);
const privateIds = new Set(
Object.values(agent.tool_resources ?? {}).flatMap((resource) => resource?.file_ids ?? []),
);
const selected = new Set(
selectCodeFileGenerations(
candidates,
privateIds,
codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
).files.flatMap((file) => (file ? [file.file_id] : [])),
);
for (const file of candidates)
if (!selected.has(file.file_id)) excludedCodeFileIds.add(file.file_id);
codeSnapshotMounts = planCodeSnapshotMounts(
candidates.filter((file) => selected.has(file.file_id)),
privateIds,
codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
);
const needed = (file: IMongoFile | TFile): boolean =>
!excludedCodeFileIds.has(file.file_id) ||
isModelBoundAttachmentFile(file) ||
(wantsSearchFiles && allowsFileResource(file, EToolResources.file_search, membership));
currentFiles = currentFiles?.filter(needed);
deferredProvisionFiles = deferredProvisionFiles.filter(needed);
}
let endpointFileType: EModelEndpoint | undefined;
if (!paramEndpoints.has(agent.endpoint ?? '')) {
@ -2097,6 +2131,8 @@ export async function initializeAgent(
adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
hasEntityEmbeddings: db.hasEntityEmbeddings,
provisionCandidates: deferredProvisionFiles as unknown as TFile[],
excludedCodeFileIds,
codeSnapshotMounts,
codeRouteKey: codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
codeBaseUrl: codeExecutionContext.baseUrl,
codeExecutionProfile: codeExecutionContext.executionProfile,

View file

@ -7,7 +7,6 @@ import {
FileSources,
getCodeEnvRefs,
canToolResourceConsume,
hasToolResourceDestination,
} from 'librechat-data-provider';
import type {
AgentToolResources,
@ -20,7 +19,12 @@ import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas';
import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose';
import type { CodeExecutionContext } from './execution';
import type { ServerRequest } from '~/types';
import {
getCodeFileDestination,
selectCodeFileGenerations,
planCodeSnapshotMounts,
} from '~/files/code/destinations';
import { allowsFileResource } from '~/files/consumers';
import { getSafeErrorMetadata } from '~/utils/errors';
import { TOOL_RESOURCE_KEYS } from './orphans';
@ -327,6 +331,7 @@ const categorizeFileForToolResources = ({
requestFileSet,
processedResourceFiles,
agentScoped = false,
codeFileIds,
agentId,
userId,
}: {
@ -336,12 +341,17 @@ const categorizeFileForToolResources = ({
processedResourceFiles: Set<string>;
/** Whether this file's vectors were embedded under the agent's entity_id. */
agentScoped?: boolean;
codeFileIds: ReadonlySet<string>;
/** The agent whose vector namespace this turn will search, when one is scoped. */
agentId?: string;
/** Owner of the unscoped namespace, which is where anything not agent-scoped lands. */
userId?: string;
}): void => {
if (file.metadata?.codeEnvRef || file.metadata?.codeEnvRefs) {
if (
file.context !== FileContext.execute_code_snapshot &&
codeFileIds.has(file.file_id) &&
(file.metadata?.codeEnvRef || file.metadata?.codeEnvRefs)
) {
addFileToResource({
file,
resourceType: EToolResources.execute_code,
@ -443,6 +453,8 @@ const withDeferredCandidates = (
const computeProvisionState = async ({
req,
attachments,
codeFileIds,
codeSnapshotMounts,
resourcePrincipal,
enabledToolResources,
tool_resources,
@ -463,6 +475,8 @@ const computeProvisionState = async ({
}: {
req?: ServerRequest;
attachments: Array<TFile>;
codeFileIds: ReadonlySet<string>;
codeSnapshotMounts: ReadonlyMap<string, string>;
agentId?: string;
codeRouteKey?: string;
/** Base URL of the deployment this turn runs on, used to probe its own refs. */
@ -488,28 +502,8 @@ const computeProvisionState = async ({
return undefined;
}
/* The legacy chooser makes the destination an explicit user decision, so a file it
* uploaded carries no reference for the destinations the user declined, and queueing on
* a missing reference would read those declines as work to do. The decision belongs to
* the file, not to this request: the endpoint deciding this turn need not be the one it
* was uploaded under. Records predating the marker were written by the chooser flow,
* so their existing resource membership/reference is the only durable destination evidence. */
const cameFromChooser = (file: TFile): boolean => {
const choice = file.metadata?.destinationChosen;
return choice == null ? true : choice;
};
/* Search intent survives vector cleanup without claiming the vectors still exist.
* A chooser's other destinations remain declined unless membership or a copy proves them. */
const allowsResource = (file: TFile, resourceType: EToolResources): boolean => {
if (!cameFromChooser(file)) {
return true;
}
const isPersistedResource =
file.file_id != null &&
persistedResourceMembership?.get(resourceType)?.has(file.file_id) === true;
return isPersistedResource || hasToolResourceDestination(file, resourceType);
};
const allowsResource = (file: TFile, resource: EToolResources): boolean =>
allowsFileResource(file, resource, persistedResourceMembership);
const provisionable = attachments.filter(
(file) =>
file != null &&
@ -603,7 +597,12 @@ const computeProvisionState = async ({
const canProbeRoute = activeCodeRouteKey === 'default' || codeBaseUrl != null;
const filesWithIdentifiers =
needsCodeEnv && checkSessionsAlive && canProbeRoute
? provisionable.filter((f) => f?.file_id && codeEnvRefForRoute(f, activeCodeRouteKey))
? provisionable.filter(
(f) =>
f.context !== FileContext.execute_code_snapshot &&
codeFileIds.has(f.file_id) &&
codeEnvRefForRoute(f, activeCodeRouteKey),
)
: [];
/** Code API auth is optional: deployments may use a legacy key, JWT bearer minting,
@ -654,6 +653,7 @@ const computeProvisionState = async ({
const codeEnvFiles: TFile[] = [];
const codeEnvRecoveryNames: NonNullable<ProvisionState['codeEnvRecoveryNames']> = new Map();
const vectorDBFiles: TFile[] = [];
const displacedCodeFileIds = new Set<string>();
for (const file of provisionable) {
if (!file?.file_id) {
@ -680,6 +680,7 @@ const computeProvisionState = async ({
* the sandbox refuses aborts the tool batch over a file meant for the model. */
if (
needsCodeEnv &&
codeFileIds.has(file.file_id) &&
allowsResource(file, EToolResources.execute_code) &&
canToolResourceConsume(EToolResources.execute_code, file.type ?? '')
) {
@ -693,7 +694,18 @@ const computeProvisionState = async ({
* added to execute_code resources, so the check runs before the processed
* guard. Clear both the legacy ref and its route entry, else getCodeEnvRefs
* keeps resolving the dead session over the re-provisioned one. */
if (isStale) {
const plannedName = codeSnapshotMounts.get(file.file_id);
const displaced =
routeRef != null &&
plannedName != null &&
(plannedName !==
(routeRef.sandboxFilename ?? getCodeFileDestination(file, activeCodeRouteKey)) ||
(file.context === FileContext.execute_code &&
plannedName !== getCodeFileDestination(file, activeCodeRouteKey)));
if (file.context === FileContext.execute_code_snapshot || displaced) {
codeEnvFiles.push(file);
if (displaced) displacedCodeFileIds.add(file.file_id);
} else if (isStale) {
if (routeRef?.sandboxFilename) {
codeEnvRecoveryNames.set(file.file_id, {
name: routeRef.sandboxFilename,
@ -749,12 +761,22 @@ const computeProvisionState = async ({
}
}
// A renamed receipt is not byte identity. Only its primary-storage reupload may mount it.
const codeResource = tool_resources[EToolResources.execute_code];
if (codeResource && displacedCodeFileIds.size > 0) {
codeResource.file_ids = codeResource.file_ids?.filter((id) => !displacedCodeFileIds.has(id));
codeResource.files = codeResource.files?.filter(
(file) => !displacedCodeFileIds.has(file.file_id),
);
}
if (codeEnvFiles.length === 0 && vectorDBFiles.length === 0) {
return undefined;
}
return {
codeEnvFiles,
codeEnvRecoveryNames,
codeEnvDestinations: new Map(codeSnapshotMounts),
vectorDBFiles,
aliveFileIds: aliveFileIds ?? new Set(),
agentScopedFileIds: new Set(scopedIds),
@ -775,6 +797,8 @@ export const primeResources = async ({
checkSessionsAlive,
loadCodeApiKey,
provisionCandidates,
excludedCodeFileIds,
codeSnapshotMounts: admittedSnapshotMounts,
screenPersistentFiles,
adoptLegacySearchEmbeddings,
hasEntityEmbeddings,
@ -802,6 +826,9 @@ export const primeResources = async ({
* provisioning only and never returned as attachments: re-delivering an earlier
* upload to the model on every later turn is not the intent. */
provisionCandidates?: Array<TFile>;
/** Pre-admission code losers may still be needed by search or prompt delivery. */
excludedCodeFileIds?: ReadonlySet<string>;
codeSnapshotMounts?: ReadonlyMap<string, string>;
/** True when this endpoint still shows the explicit upload-destination chooser. */
/** Applies the caller's endpoint and content policies. Persistent agent files are read
* here rather than by the caller, so the caller has no chance to screen them itself
@ -964,35 +991,77 @@ export const primeResources = async ({
agentId,
});
}
}
if (screenPersistentFiles) {
const admitted = screenPersistentFiles(persistedResourceFiles);
/* The tool primers read tool_resources ids directly and re-check only access, so
* a record dropped here still reaches the Code API or RAG unless its id goes with
* it. Screening the hydrated list alone screens nothing. */
if (admitted.length !== persistedResourceFiles.length) {
const admittedIds = new Set(
admitted.map((file) => file.file_id).filter((id): id is string => id != null),
);
const rejectedIds = persistedResourceFiles
.map((file) => file.file_id)
.filter((id): id is string => id != null && !admittedIds.has(id));
const runtimeResources = [
tool_resources[EToolResources.execute_code],
tool_resources[EToolResources.file_search],
];
for (const resource of runtimeResources) {
if (resource?.file_ids == null) {
continue;
}
resource.file_ids = resource.file_ids.filter((id: string) => !rejectedIds.includes(id));
}
for (const id of rejectedIds) {
toolResourceFileIds.delete(id);
const files = _attachments ? await _attachments : [];
const codeCandidates = withDeferredCandidates(
persistedResourceFiles,
withDeferredCandidates(
files.filter((file): file is TFile => file != null),
provisionCandidates,
),
);
const eligibleCodeFiles = codeCandidates.filter(
(file) =>
!excludedCodeFileIds?.has(file.file_id) &&
allowsFileResource(file, EToolResources.execute_code, persistedResourceMembership),
);
const codeFileIds = new Set(
selectCodeFileGenerations(
eligibleCodeFiles,
persistedResourceFileIds,
codeRouteKey,
).files.flatMap((file) => (file ? [file.file_id] : [])),
);
const codeSnapshotMounts = planCodeSnapshotMounts(
eligibleCodeFiles.filter((file) => codeFileIds.has(file.file_id)),
persistedResourceFileIds,
codeRouteKey,
admittedSnapshotMounts,
);
const supersededCodeFileIds = new Set(
codeCandidates.filter((file) => !codeFileIds.has(file.file_id)).map((file) => file.file_id),
);
const codeResource = tool_resources[EToolResources.execute_code];
if (codeResource?.file_ids) {
codeResource.file_ids = codeResource.file_ids.filter((id) => !supersededCodeFileIds.has(id));
}
persistedResourceFiles = persistedResourceFiles.filter(
(file) =>
codeFileIds.has(file.file_id) ||
(enabledToolResources?.has(EToolResources.file_search) &&
allowsFileResource(file, EToolResources.file_search, persistedResourceMembership)) ||
contextFileIds.has(file.file_id) ||
imageEditFileIdSet.has(file.file_id),
);
if (screenPersistentFiles && persistedResourceFileIds.size > 0) {
const admitted = screenPersistentFiles(persistedResourceFiles);
/* The tool primers read tool_resources ids directly and re-check only access, so
* a record dropped here still reaches the Code API or RAG unless its id goes with
* it. Screening the hydrated list alone screens nothing. */
if (admitted.length !== persistedResourceFiles.length) {
const admittedIds = new Set(
admitted.map((file) => file.file_id).filter((id): id is string => id != null),
);
const rejectedIds = persistedResourceFiles
.map((file) => file.file_id)
.filter((id): id is string => id != null && !admittedIds.has(id));
const runtimeResources = [
tool_resources[EToolResources.execute_code],
tool_resources[EToolResources.file_search],
];
for (const resource of runtimeResources) {
if (resource?.file_ids == null) {
continue;
}
resource.file_ids = resource.file_ids.filter((id: string) => !rejectedIds.includes(id));
}
for (const id of rejectedIds) {
toolResourceFileIds.delete(id);
}
persistedResourceFiles = admitted;
}
persistedResourceFiles = admitted;
}
for (const file of persistedResourceFiles) {
@ -1008,6 +1077,7 @@ export const primeResources = async ({
tool_resources,
requestFileSet,
processedResourceFiles,
codeFileIds,
agentScoped: agentId != null && isAgentScopedFile(file),
agentId,
userId: resourcePrincipal?.id,
@ -1042,6 +1112,8 @@ export const primeResources = async ({
...(provisionCandidates ?? []),
...resourceProvisionCandidates,
]),
codeFileIds,
codeSnapshotMounts,
resourcePrincipal,
enabledToolResources,
tool_resources,
@ -1071,7 +1143,6 @@ export const primeResources = async ({
};
}
const files = await _attachments;
const requestAttachmentFileIds = new Set<string>();
for (const file of files) {
@ -1084,6 +1155,7 @@ export const primeResources = async ({
tool_resources,
requestFileSet,
processedResourceFiles,
codeFileIds,
userId: resourcePrincipal?.id,
});
@ -1111,6 +1183,8 @@ export const primeResources = async ({
...(provisionCandidates ?? []),
...resourceProvisionCandidates,
]),
codeFileIds,
codeSnapshotMounts,
resourcePrincipal,
enabledToolResources,
tool_resources,

View file

@ -1,3 +1,4 @@
import { FileContext } from 'librechat-data-provider';
import type { CodeEnvRef } from 'librechat-data-provider';
import type { ServerRequest } from '~/types';
import { createFileProvisionRecovery } from '../operations';
@ -166,3 +167,27 @@ it.each([
).rejects.toBe(error);
expect(clearFileCodeEnvRefs).not.toHaveBeenCalled();
});
it.each([true, false])(
'reconciles snapshot receipts without deleting mutable paths (CAS %s)',
async (matched) => {
const { httpClient, clearFileCodeEnvRefs, remove } = setup();
clearFileCodeEnvRefs.mockResolvedValue(matched);
const file = {
file_id: 'snapshot-v1',
user: 'owner',
tenantId: 'tenant-a',
context: FileContext.execute_code_snapshot,
metadata: { codeEnvRef: ref, codeEnvRefs: { default: ref } },
};
if (matched) await expect(remove(req, file)).resolves.toEqual({ deleted: true });
else await expect(remove(req, file)).rejects.toThrow('Code storage reconciliation failed');
expect(httpClient).not.toHaveBeenCalled();
expect(clearFileCodeEnvRefs).toHaveBeenCalledWith({
file_id: 'snapshot-v1',
userId: 'owner',
tenantId: 'tenant-a',
refs: [{ routeKey: 'default', ref }],
});
},
);

View file

@ -1,4 +1,4 @@
import { EModelEndpoint, getCodeEnvRefs } from 'librechat-data-provider';
import { FileContext, EModelEndpoint, getCodeEnvRefs } from 'librechat-data-provider';
import type { ClearFileCodeEnvRefsParams } from '@librechat/data-schemas';
import type { CodeEnvReferenceSet } from 'librechat-data-provider';
import type { AxiosRequestConfig } from 'axios';
@ -39,6 +39,7 @@ export function createCodeFileDeleter({
req: ServerRequest,
file: {
file_id?: string;
context?: string;
user?: { toString(): string };
tenantId?: string;
metadata?: CodeEnvReferenceSet;
@ -47,6 +48,21 @@ export function createCodeFileDeleter({
return async (req, file) => {
const refs = getCodeEnvRefs(file.metadata);
if (refs.length === 0) return undefined;
// Recovery receipts name shared paths. Reconcile only this record, never the live object.
if (file.context === FileContext.execute_code_snapshot) {
if (
!file.file_id ||
!(await clearFileCodeEnvRefs({
file_id: file.file_id,
userId: file.user?.toString() ?? req.user?.id ?? '',
tenantId: file.tenantId,
refs: refs.map(([routeKey, ref]) => ({ routeKey, ref })),
}))
) {
throw new Error('Code storage reconciliation failed');
}
return { deleted: true };
}
let skipped = false;
for (const [executionRouteKey, ref] of refs) {
const executionProfile = ref.executionProfile ?? 'default';

View file

@ -1,5 +1,9 @@
import { FileContext } from 'librechat-data-provider';
import {
claimCodeDestination,
selectCodeFileGenerations,
getCodeFileDestination,
planCodeSnapshotMounts,
reserveCodeDestination,
createCodeDestinationSet,
sortCodeFilesByDestinationPriority,
@ -305,3 +309,105 @@ describe('sortCodeFilesByDestinationPriority', () => {
expect(input).toEqual([older, newer]);
});
});
describe('scoped generated destination selection', () => {
const snapshot = (file_id: string, time: number, overrides = {}) => ({
file_id,
user: 'owner',
tenantId: 'tenant-a',
filename: 'nested/report.txt',
type: 'text/plain',
context: FileContext.execute_code_snapshot,
metadata: { sourceDispatchedAt: time },
...overrides,
});
it('arbitrates a rewritten upload alias at its real destination', () => {
const upload = {
...snapshot('upload', 50),
context: FileContext.message_attachment,
filename: 'display.txt',
createdAt: new Date(50),
metadata: {
codeEnvRef: {
kind: 'user' as const,
id: 'owner',
storage_session_id: 'session',
file_id: 'object',
sandboxFilename: 'nested/report.txt',
},
},
};
expect(
selectCodeFileGenerations([upload, snapshot('v2', 200)]).files.map((file) => file?.file_id),
).toEqual(['v2']);
});
it('never lets tenant or private generations supersede the shared owner generation', () => {
const result = selectCodeFileGenerations(
[
snapshot('v1', 100),
snapshot('v2', 200),
snapshot('tenant-b', 300, { tenantId: 'tenant-b' }),
snapshot('private', 400),
],
new Set(['private']),
);
expect(result.files.map((file) => file?.file_id)).toEqual(['tenant-b', 'v2', 'private']);
});
it('does not treat a mutable receipt alias as a snapshot identity', () => {
const file = snapshot('v1', 100, {
metadata: {
codeEnvRef: {
kind: 'user',
id: 'owner',
storage_session_id: 'session',
file_id: 'object',
sandboxFilename: 'wrong-alias.txt',
},
},
});
expect(getCodeFileDestination(file)).toBe('nested/report.txt');
});
});
describe('legacy capture mount preferences', () => {
it.each([
['nested/chart.png', 'nested/chart.webp', 'image/webp', 'nested/chart.webp'],
['report.txt', 'report-assigned.txt', 'text/plain', 'report.txt'],
])(
'preserves the turn plan but not persistent alias lineage for %s',
(capture, name, type, canonical) => {
const legacy = {
file_id: 'legacy',
filename: capture,
type,
context: FileContext.execute_code,
metadata: {
codeEnvRef: {
kind: 'user' as const,
id: 'owner',
storage_session_id: 'session',
file_id: 'object',
sandboxFilename: name,
},
},
};
const literal = {
file_id: 'literal',
filename: name,
type,
context: FileContext.execute_code_snapshot,
};
const admitted = planCodeSnapshotMounts([legacy, literal]);
const alias = admitted.get(legacy.file_id)!;
expect(admitted.get(literal.file_id)).toBe(name);
expect(alias).not.toBe(name);
expect(
planCodeSnapshotMounts([legacy], undefined, 'default', admitted).get(legacy.file_id),
).toBe(alias);
legacy.metadata.codeEnvRef.sandboxFilename = alias;
expect(planCodeSnapshotMounts([legacy]).get(legacy.file_id)).toBe(canonical);
legacy.metadata.codeEnvRef.sandboxFilename = name;
expect(planCodeSnapshotMounts([legacy]).get(legacy.file_id)).toBe(canonical);
},
);
});

View file

@ -1,9 +1,16 @@
import {
FileContext,
getCodeEnvRefForProfile,
resolveSandboxFilename,
} from 'librechat-data-provider';
import type { CodeEnvReferenceSet } from 'librechat-data-provider';
import {
appendLeafSuffix,
flattenArtifactPath,
deterministicHexSuffix,
FILENAME_SEGMENT_MAX_BYTES,
} from '~/utils/files';
import { getCodeEnvUploadFilename } from './form';
/**
* Sandbox input files mount at a destination derived from their `name`, and
@ -156,6 +163,7 @@ export function reserveCodeDestination(set: CodeDestinationSet, name: string): b
}
interface CodeDestinationCandidate {
context?: string;
file_id?: string;
createdAt?: Date | string | number;
/** Last content write for a reused code-output record. `processCodeOutput`
@ -174,6 +182,12 @@ function toTime(value: Date | string | number | undefined): number {
}
function contentTime(file: CodeDestinationCandidate | null | undefined): number {
if (
file?.context === FileContext.execute_code_snapshot &&
file.metadata?.sourceDispatchedAt != null
) {
return toTime(file.metadata.sourceDispatchedAt);
}
return Math.max(toTime(file?.metadata?.sourceDispatchedAt), toTime(file?.createdAt));
}
@ -220,3 +234,154 @@ export function sortCodeFilesByDestinationPriority<T extends CodeDestinationCand
return (a?.file_id ?? '').localeCompare(b?.file_id ?? '');
});
}
type CodePathCandidate = CodeDestinationCandidate & {
filename?: string;
tenantId?: string;
user?: { toString(): string };
type?: string;
metadata?: (CodeEnvReferenceSet & { sourceDispatchedAt?: number }) | null;
};
/** Mount naming only. Captured write paths, not converted names, identify versions. */
export function getCodeFileDestination(file: CodePathCandidate, routeKey = 'default'): string {
if (
file.context !== FileContext.execute_code_snapshot &&
file.context !== FileContext.execute_code
) {
const stored = getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename;
if (stored != null) return stored;
}
const normalized = getCodeEnvUploadFilename(
resolveSandboxFilename(file.filename ?? '', file.type ?? ''),
);
// A converted artifact's receipt is a mount-name hint only, never byte equivalence.
if (file.context === FileContext.execute_code_snapshot && normalized !== file.filename) {
return (
getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename ?? normalized
);
}
return normalized;
}
/** Select captured versions among code-eligible inputs before mount conversion. */
export function selectCodeFileGenerations<T extends CodePathCandidate>(
files: Array<T | null | undefined>,
privateFileIds?: ReadonlySet<string>,
routeKey = 'default',
): { files: Array<T | null | undefined>; skippedSuperseded: number } {
const destinations = new Map<string, T>();
const generatedPaths = new Set<string>();
let skippedSuperseded = 0;
const ordered = sortCodeFilesByDestinationPriority(files, privateFileIds);
const keyFor = (file: T): string => {
const generated =
file.context === FileContext.execute_code_snapshot ||
file.context === FileContext.execute_code;
const destination = generated
? file.filename
: (getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename ??
getCodeEnvUploadFilename(resolveSandboxFilename(file.filename ?? '', file.type ?? '')));
return JSON.stringify([
privateFileIds?.has(file.file_id ?? '') ?? false,
file.tenantId ?? null,
file.user?.toString() ?? null,
destination,
]);
};
for (const file of ordered) {
if (!file) continue;
const key = keyFor(file);
if (!destinations.has(key)) destinations.set(key, file);
if (
file.context === FileContext.execute_code_snapshot ||
file.context === FileContext.execute_code
)
generatedPaths.add(key);
}
const selected = ordered.filter((file) => {
if (!file) return true;
const key = keyFor(file);
// Distinct uploads remain independent unless a generated write owns this path.
if (!generatedPaths.has(key) || destinations.get(key) === file) return true;
skippedSuperseded++;
return false;
});
return { files: selected, skippedSuperseded };
}
/** Within each scope, literal snapshots precede legacy captures and converted snapshots. */
export function planCodeSnapshotMounts<T extends CodePathCandidate>(
files: Array<T | null | undefined>,
privateFileIds?: ReadonlySet<string>,
routeKey = 'default',
preferred?: ReadonlyMap<string, string>,
): Map<string, string> {
const destinations = createCodeDestinationSet();
const ordered = sortCodeFilesByDestinationPriority(files, privateFileIds);
const snapshots: T[] = [];
const receipts: T[] = [];
const legacyCaptures: T[] = [];
const isConverted = (file: T): boolean =>
getCodeEnvUploadFilename(resolveSandboxFilename(file.filename ?? '', file.type ?? '')) !==
file.filename;
const seen = new Set<string>();
for (const file of ordered) {
if (!file || (file.file_id && seen.has(file.file_id))) continue;
if (file.file_id) seen.add(file.file_id);
if (file.context === FileContext.execute_code_snapshot) snapshots.push(file);
else if (file.context === FileContext.execute_code) legacyCaptures.push(file);
else receipts.push(file);
}
const priority = (file: T): number => {
if (!isConverted(file)) return 0;
const assigned =
preferred?.has(file.file_id ?? '') ||
getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename;
return assigned ? 1 : 2;
};
snapshots.sort((a, b) => priority(a) - priority(b));
const names = new Map<string, string>();
for (const privateScope of [false, true]) {
const inScope = (file: T): boolean =>
(privateFileIds?.has(file.file_id ?? '') ?? false) === privateScope;
for (const file of receipts) {
if (!inScope(file)) continue;
const ref = getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey);
const name =
ref?.sandboxFilename ??
(privateScope && ref != null ? getCodeFileDestination(file, routeKey) : undefined);
if (!name) continue;
if (!privateScope) {
reserveCodeDestination(destinations, name);
continue;
}
const destination = claimCodeDestination(destinations, name, file.file_id ?? name);
if (file.file_id && destination !== name) names.set(file.file_id, destination);
}
const claimSnapshot = (file: T): void => {
if (!file.file_id || !inScope(file)) return;
names.set(
file.file_id,
claimCodeDestination(
destinations,
preferred?.get(file.file_id) ?? getCodeFileDestination(file, routeKey),
file.file_id,
),
);
};
for (const file of snapshots) {
if (!isConverted(file)) claimSnapshot(file);
}
for (const file of legacyCaptures) {
if (!file.file_id || !inScope(file)) continue;
// Retain this turn's admission plan, not an alias persisted on another branch.
const name = preferred?.get(file.file_id) ?? getCodeFileDestination(file, routeKey);
names.set(file.file_id, claimCodeDestination(destinations, name, file.file_id));
}
for (const file of snapshots) {
if (isConverted(file)) claimSnapshot(file);
}
}
return names;
}

View file

@ -16,6 +16,7 @@ import {
const file = (id: string, name: string, ref: Partial<CodeEnvRef> = {}, time = 1): TFile =>
({
file_id: id,
user: 'user',
filename: name,
type: 'text/csv',
createdAt: new Date(time),
@ -27,6 +28,127 @@ const file = (id: string, name: string, ref: Partial<CodeEnvRef> = {}, time = 1)
const fresh = { lastModified: new Date().toISOString() };
describe('selectCodeFiles', () => {
it.each(['default', 'stateful'] as const)(
'reuses a fresh unnamed legacy ref with timestamp-only metadata on %s',
async (routeKey) => {
const legacy = {
...file('legacy', 'rows.csv', { executionProfile: routeKey }),
context: FileContext.execute_code,
};
const getFileInfo = jest.fn(async () => fresh);
const result = await selectCodeFiles({ files: [legacy], routeKey, getFileInfo });
expect(result.selected).toHaveLength(1);
expect(result.selected[0]).toMatchObject({
ref: legacy.metadata!.codeEnvRef,
sourceRef: legacy.metadata!.codeEnvRef,
sandboxName: 'rows.csv',
isActive: true,
});
expect(result.skippedNoRef).toBe(0);
expect(result.skippedSuperseded).toBe(0);
await expect(result.selected[0].getUploadTime()).resolves.toBe(fresh.lastModified);
await expect(result.selected[0].getUploadTime()).resolves.toBe(fresh.lastModified);
expect(getFileInfo).toHaveBeenCalledTimes(1);
},
);
it('keeps the newest snapshot at the live path across legacy records and route recovery', async () => {
const legacy = {
...file('legacy', 'rows.csv', { sandboxFilename: 'rows.csv' }),
context: FileContext.execute_code,
};
const snapshot = {
...file('snapshot', 'rows.csv', {}, 2),
context: FileContext.execute_code_snapshot,
};
snapshot.metadata = { codeEnvSource: snapshot.metadata?.codeEnvRef, sourceDispatchedAt: 2 };
const result = await selectCodeFiles({
files: [legacy, snapshot],
routeKey: 'stateful',
getFileInfo: async () => fresh,
});
expect(result.selected.map(({ file }) => file.file_id)).toEqual(['snapshot']);
expect(result.selected[0].sandboxName).toBe('rows.csv');
expect(result.selected[0].ref).toBeUndefined();
expect(result.skippedSuperseded).toBe(1);
});
describe.each([
{
label: 'converted image',
capture: 'nested/chart.png',
name: 'nested/chart.webp',
canonical: 'nested/chart.webp',
type: 'image/webp',
},
{
label: 'assigned text alias',
capture: 'report.txt',
name: 'report-assigned.txt',
canonical: 'report.txt',
type: 'text/plain',
},
])('$label', ({ capture, name, canonical, type }) => {
it.each([
['default', false],
['default', true],
['stateful', false],
['stateful', true],
] as const)(
'keeps literal snapshot paths ahead of legacy receipts on %s (stale=%s)',
async (routeKey, stale) => {
const legacy = {
...file('legacy', capture, {
sandboxFilename: name,
executionProfile: routeKey,
}),
type,
context: FileContext.execute_code,
};
const snapshot = {
...file('snapshot', name, {}, 2),
type,
context: FileContext.execute_code_snapshot,
};
const result = await selectCodeFiles({
files: [legacy, snapshot],
routeKey,
getFileInfo: async () => (stale ? null : fresh),
});
expect(result.selected).toHaveLength(2);
expect(result.skippedSuperseded).toBe(0);
expect(result.selected.find((entry) => entry.file.file_id === 'snapshot')).toMatchObject({
sandboxName: name,
isActive: false,
});
expect(result.selected.find((entry) => entry.file.file_id === 'legacy')).toMatchObject({
sandboxName:
canonical === name ? expect.stringMatching(/^nested\/chart-.+\.webp$/) : canonical,
isActive: false,
sourceRef: { file_id: 'legacy' },
});
const alias = result.selected.find((entry) => entry.file.file_id === 'legacy')!.sandboxName;
legacy.metadata!.codeEnvRef!.sandboxFilename = alias;
const repeated = await selectCodeFiles({
files: [legacy, snapshot],
routeKey,
getFileInfo: async () => (stale ? null : fresh),
});
expect(repeated.selected.find((entry) => entry.file.file_id === 'legacy')).toMatchObject({
sandboxName: alias,
isActive: !stale && alias === canonical,
});
const oldBranch = await selectCodeFiles({
files: [legacy],
routeKey,
getFileInfo: async () => (stale ? null : fresh),
});
expect(oldBranch.selected[0].sandboxName).toBe(canonical);
},
);
});
it('preserves provisioned aliases across turns and reuses legacy metadata probes', async () => {
const getFileInfo = jest.fn(async (ref: CodeEnvRef) => ({
...fresh,
@ -125,10 +247,17 @@ describe('selectCodeFiles', () => {
});
expect(recovered.selected).toHaveLength(2);
expect(new Set(recovered.selected.map((f) => f.sandboxName)).size).toBe(2);
const restored = recovered.selected.map(({ file: original, sandboxName, sourceRef }) => ({
...original,
metadata: { codeEnvRef: { ...sourceRef, sandboxFilename: sandboxName } },
}));
const restored = recovered.selected.map(({ file: original, sandboxName, sourceRef }) => {
if (!sourceRef.file_id || !sourceRef.storage_session_id)
throw new Error('Expected owned legacy copy');
const ref: CodeEnvRef = {
...sourceRef,
file_id: sourceRef.file_id,
storage_session_id: sourceRef.storage_session_id,
sandboxFilename: sandboxName,
};
return { ...original, metadata: { codeEnvRef: ref } };
});
const next = await selectCodeFiles({
files: restored,
routeKey: 'default',
@ -205,6 +334,39 @@ describe('selectCodeFiles', () => {
);
});
it.each([
['default', true],
['default', false],
['stateful', true],
['stateful', false],
] as const)(
'reuploads a displaced private receipt rather than trusting its live bytes on %s (named %s)',
async (routeKey, named) => {
const shared = { ...file('shared', 'rows.csv'), context: FileContext.execute_code_snapshot };
const privateFile = file('private', 'rows.csv', {
kind: 'agent',
id: 'agent-a',
sandboxFilename: named ? 'rows.csv' : undefined,
executionProfile: routeKey,
});
const result = await selectCodeFiles({
files: [privateFile, shared],
privateFileIds: new Set(['private']),
routeKey,
getFileInfo: async () => ({ ...fresh, originalFilename: 'rows.csv' }),
});
expect(result.selected).toHaveLength(2);
expect(result.selected.find((entry) => entry.file.file_id === 'shared')?.sandboxName).toBe(
'rows.csv',
);
expect(result.selected.find((entry) => entry.file.file_id === 'private')).toMatchObject({
sandboxName: expect.stringMatching(/^rows-.+\.csv$/),
isActive: false,
sourceRef: { kind: 'agent', id: 'agent-a', file_id: 'private' },
});
},
);
it('keeps the freshness decision paired with the selected image destination', async () => {
const now = Date.parse('2026-09-10T12:00:00Z');
const clock = jest.spyOn(Date, 'now').mockReturnValue(now);

View file

@ -16,7 +16,9 @@ import {
claimCodeDestination,
createCodeDestinationSet,
reserveCodeDestination,
sortCodeFilesByDestinationPriority,
selectCodeFileGenerations,
getCodeFileDestination,
planCodeSnapshotMounts,
} from './destinations';
import {
codeServerHttpAgent,
@ -92,10 +94,18 @@ export function checkCodeFileActive(dateString: string | undefined): boolean {
return dateString != null && (Date.now() - new Date(dateString).getTime()) / 3_600_000 < 23;
}
interface SnapshotCodeOwner {
kind: 'user';
id: string;
storage_session_id?: undefined;
file_id?: undefined;
sandboxFilename?: undefined;
}
interface PrimedCodeFile {
file: TFile;
ref: CodeEnvRef | undefined;
sourceRef: CodeEnvRef;
sourceRef: CodeEnvRef | SnapshotCodeOwner;
sandboxName: string;
isActive: boolean;
getUploadTime: () => Promise<string | undefined>;
@ -118,15 +128,28 @@ export async function selectCodeFiles({
signal?: AbortSignal;
}): Promise<{ selected: PrimedCodeFile[]; skippedNoRef: number; skippedSuperseded: number }> {
let skippedNoRef = 0;
let skippedSuperseded = 0;
const generations = selectCodeFileGenerations(files, privateFileIds, routeKey);
let skippedSuperseded = generations.skippedSuperseded;
const snapshotMounts = planCodeSnapshotMounts(generations.files, privateFileIds, routeKey);
const candidates = [];
const registry = createCodeApiUploadRegistry();
for (const file of sortCodeFilesByDestinationPriority(files, privateFileIds)) {
for (const file of generations.files) {
if (!file) {
continue;
}
const ref = getCodeEnvRefForProfile(file.metadata, routeKey);
const sourceRef = ref ?? getCodeEnvRefs(file.metadata)[0]?.[1];
const snapshot = file.context === FileContext.execute_code_snapshot;
const ownedRef = getCodeEnvRefForProfile(file.metadata, routeKey);
// Snapshot receipts can name mounts, but never prove their current bytes.
const ref = snapshot ? undefined : ownedRef;
const sourceRef =
ownedRef ??
getCodeEnvRefs(file.metadata)[0]?.[1] ??
(snapshot
? {
kind: 'user' as const,
id: file.user.toString(),
}
: undefined);
if (!sourceRef) {
skippedNoRef++;
continue;
@ -138,7 +161,7 @@ export async function selectCodeFiles({
scope: routeKey,
concurrency,
signal,
task: () => getFileInfo(sourceRef),
task: () => (ref ? getFileInfo(ref) : Promise.resolve(null)),
}));
candidates.push({ file, ref, sourceRef, getInfo });
}
@ -147,9 +170,19 @@ export async function selectCodeFiles({
const { file, ref, sourceRef, getInfo } = candidate;
const info = ref ? await getInfo() : null;
signal?.throwIfAborted();
const recovering = !ref || !checkCodeFileActive(info?.lastModified);
const storedName = sourceRef.sandboxFilename ?? info?.originalFilename;
const sandboxName = storedName ?? resolveSandboxFilename(file.filename, file.type);
const plannedName = snapshotMounts.get(file.file_id);
const recovering =
!ref ||
!checkCodeFileActive(info?.lastModified) ||
(plannedName != null &&
(plannedName !== (storedName ?? getCodeFileDestination(file, routeKey)) ||
(file.context === FileContext.execute_code &&
plannedName !== getCodeFileDestination(file, routeKey))));
const sandboxName =
file.context === FileContext.execute_code_snapshot
? (snapshotMounts.get(file.file_id) ?? getCodeFileDestination(file, routeKey))
: (plannedName ?? storedName ?? resolveSandboxFilename(file.filename, file.type));
const destination = recovering
? getCodeEnvUploadFilename(resolveSandboxFilename(sandboxName, file.type))
: sandboxName;
@ -287,7 +320,11 @@ export function getCodeFileContextLine(
/** Whether the code-file note mentions this file at `destination` at all. */
export function isDescribedCodeFile(file: CodeContextFile, destination: string): boolean {
return file.context !== FileContext.execute_code || destination !== file.filename;
return (
(file.context !== FileContext.execute_code &&
file.context !== FileContext.execute_code_snapshot) ||
destination !== file.filename
);
}
/** Provenance and preview state appended to a file shown under `shownName`. */
@ -297,7 +334,7 @@ export function getCodeFileContextSuffix(
shownName: string,
): string {
const origin =
file.context === FileContext.execute_code
file.context === FileContext.execute_code || file.context === FileContext.execute_code_snapshot
? ` (written earlier as ${file.filename})`
: `${agentScoped ? '' : ' (attached by user)'}${
shownName !== file.filename ? ` (uploaded as ${file.filename})` : ''

View file

@ -0,0 +1,990 @@
import os from 'node:os';
import sharp from 'sharp';
import path from 'node:path';
import mongoose from 'mongoose';
import { Readable } from 'node:stream';
import { createReadStream } from 'node:fs';
import { mockClient } from 'aws-sdk-client-mock';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { buffer as readStream } from 'node:stream/consumers';
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import { createMethods, fileSchema } from '@librechat/data-schemas';
import {
FileContext,
FileSources,
EToolResources,
getEndpointFileConfig,
mergeFileConfig,
} from 'librechat-data-provider';
import type { CodeFileCommitData, IMongoFile } from '@librechat/data-schemas';
import type { TFile } from 'librechat-data-provider';
import type { CodeOutputProcessorDeps } from './process';
import type { TGetFiles } from '~/agents/resources';
import type { CodeFileAgent } from './queued';
import type { ServerRequest } from '~/types';
import { hasOfficeHtmlPath, extractCodeArtifactText, getExtractedTextFormat } from './extract';
import { createLocalFileDeleter, saveLocalBuffer, writeLocalFile } from '~/storage/local';
import { createFileProvisionRecovery, withFileProvisioning } from '../operations';
import { sanitizeArtifactPath, flattenArtifactPath } from '~/utils/files';
import { createProvisionFilesCallback } from '../provision/callback';
import { fromLocalUrlPath, stripCacheBust } from '~/storage/path';
import { createDeleteFileWithSecondaryStorage } from '../storage';
import { createProvisionService } from '../provision/service';
import { saveBufferToS3, getS3Key } from '~/storage/s3/crud';
import { createCodeApiUploadRegistry } from '~/utils/code';
import { getStorageMetadata } from '~/storage/metadata';
import { prepareQueuedCodeFileContext } from './queued';
import { primeResources } from '~/agents/resources';
import { classifyCodeArtifact } from './classify';
import { createCodeFileDeleter } from './delete';
import { processCodeOutput } from './process';
import { selectCodeFiles } from './priming';
import * as s3 from '~/cdn/s3';
/** Real Mongo claims, provisioning and filesystem bytes, including image conversion. */
describe('Generated file snapshots', () => {
let mongo: MongoMemoryServer;
let directory: string;
let methods: ReturnType<typeof createMethods>;
let deps: CodeOutputProcessorDeps;
let req: ServerRequest & Required<Pick<ServerRequest, 'user' | 'config'>>;
const user = new mongoose.Types.ObjectId().toString();
const paths = () => ({ publicPath: directory, uploads: path.join(directory, 'uploads') });
beforeAll(async () => {
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
await mongoose.connect(mongo.getUri());
const File = mongoose.model('File', fileSchema);
await File.init();
methods = createMethods(mongoose);
});
beforeEach(async () => {
await mongoose.models.File.deleteMany({});
directory = await mkdtemp(path.join(os.tmpdir(), 'generated-snapshots-'));
req = {
user: { id: user, tenantId: 'tenant-a' },
app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } },
config: { fileStrategy: FileSources.local, imageOutputType: 'webp' },
} as unknown as typeof req;
const saveBuffer: NonNullable<
ReturnType<CodeOutputProcessorDeps['getStrategyFunctions']>['saveBuffer']
> = (input) => saveLocalBuffer({ ...input, paths: paths(), basePath: 'uploads' });
deps = {
fileStorage: { ...methods, recovery: createFileProvisionRecovery() },
claimCodeFile: methods.claimCodeFile,
commitCodeFile: methods.commitCodeFile,
getCodeOutputFileSettings: () => ({
endpointFileConfig: getEndpointFileConfig({
fileConfig: mergeFileConfig({}),
endpoint: 'agents',
}),
fileSizeLimit: 1024 * 1024,
}),
downloadCodeOutputBuffer: async () => {
throw new Error('Fixture requires prepared bytes');
},
createDownloadFallback: (input) => ({
filename: input.name,
filepath: 'fallback',
conversationId: input.conversationId,
expiresAt: input.expiresAt,
}),
getRetentionExpiry: async () => ({}),
convertImage: async (_req, input, _detail, filename) => {
const buffer = await sharp(input).webp({ lossless: true }).toBuffer();
return {
filepath: await saveBuffer({
userId: user,
tenantId: req.user.tenantId,
buffer,
fileName: filename,
basePath: 'uploads',
}),
bytes: buffer.length,
};
},
getStrategyFunctions: () => ({ saveBuffer }),
determineFileType: async () => undefined,
finalizePreview: async () => null,
hasOfficeHtmlPath,
sanitizeArtifactPath,
flattenArtifactPath,
classifyCodeArtifact,
extractCodeArtifactText,
getExtractedTextFormat,
getStorageMetadata,
logAxiosError: ({ error }) => {
throw error;
},
logger: { warn: () => undefined, debug: () => undefined },
};
});
afterEach(async () => {
await rm(directory, { recursive: true, force: true });
});
afterAll(async () => {
await mongoose.disconnect();
mongoose.deleteModel('File');
await mongo?.stop();
});
async function emit(buffer: Buffer, messageId: string, name = 'report.txt', stamp?: number) {
const result = await processCodeOutput(
{
req,
id: 'same-live-object',
session_id: 'same-live-session',
name,
messageId,
toolCallId: `tool-${messageId}`,
conversationId: 'conversation-a',
preparedBuffer: buffer,
freshClaimAfter: stamp,
},
deps,
);
const file = result?.file;
if (!file || !('file_id' in file)) throw new Error('Expected a stored snapshot');
return file;
}
async function reload(file_id: string) {
const file = await mongoose.models.File.findOne({ file_id }).lean<CodeFileCommitData>();
if (!file) throw new Error('Snapshot disappeared on reload');
const bytes = await readFile(
file.storageKey
? path.join(directory, 'objects', file.storageKey)
: path.join(directory, fromLocalUrlPath(stripCacheBust(file.filepath))),
);
return { file, bytes };
}
function primingFile(file: IMongoFile): TFile {
return {
file_id: file.file_id,
user: file.user.toString(),
tenantId: file.tenantId,
filename: file.filename,
filepath: file.filepath,
type: file.type,
bytes: file.bytes,
object: 'file',
embedded: file.embedded === true,
usage: file.usage ?? 1,
context: file.context as FileContext,
metadata: file.metadata,
createdAt: file.createdAt,
source: file.source as FileSources,
};
}
async function initializeCode(
ids: string[],
privateIds: string[] = [],
deferred = true,
failOnceAt?: string,
stale = false,
) {
const files = (
(await methods.getFiles({ file_id: { $in: ids }, user, tenantId: req.user.tenantId })) ?? []
).map(primingFile);
const getFiles = jest.fn<ReturnType<TGetFiles>, Parameters<TGetFiles>>(async (filter) =>
((await methods.getFiles({ ...filter, user, tenantId: req.user.tenantId })) ?? []).map(
primingFile,
),
);
const liveDirectory = path.join(directory, 'sandbox');
const upload = jest.fn(async ({ stream, filename, onUploadStart }: Record<string, unknown>) => {
if (!(stream instanceof Readable) || typeof filename !== 'string')
throw new Error('Invalid upload');
if (filename === failOnceAt) {
failOnceAt = undefined;
throw Object.assign(new Error('Upload rejected'), { response: { status: 400 } });
}
if (typeof onUploadStart === 'function') onUploadStart();
await writeLocalFile(
path.join(liveDirectory, path.dirname(filename)),
path.basename(filename),
await readStream(stream),
);
return {
storage_session_id: 'same-live-session',
file_id: filename === 'report.txt' ? 'same-live-object' : filename,
filename,
};
});
const service = createProvisionService({
getStrategyFunctions: (source) =>
source === FileSources.execute_code
? { handleFileUpload: upload }
: {
getDownloadStream: async (_req, filepath) =>
createReadStream(path.join(directory, fromLocalUrlPath(stripCacheBust(filepath)))),
},
uploadVectors: async () => ({ embedded: true }),
loadAuthValues: async () => ({}),
});
const checkSessionsAlive = jest.fn(service.checkSessionsAlive);
const primed = await primeResources({
req,
getFiles,
requestFileSet: new Set(),
attachments: deferred ? undefined : Promise.resolve(files),
provisionCandidates: deferred ? files : undefined,
tool_resources: privateIds.length ? { execute_code: { file_ids: privateIds } } : {},
agentId: 'agent-a',
enabledToolResources: new Set([EToolResources.execute_code]),
checkSessionsAlive,
});
const context: CodeFileAgent = {
id: 'agent-a',
tool_resources: primed.tool_resources,
provisionState: primed.provisionState,
fileConsumers: { executeCode: true, fileSearch: false },
};
// Eager tool loading runs before the lazy callback, as initializeAgent does.
const eager = await selectCodeFiles({
files: context.tool_resources?.execute_code?.files ?? [],
privateFileIds: new Set(privateIds),
routeKey: 'default',
getFileInfo: async () => (stale ? null : { lastModified: new Date().toISOString() }),
});
for (const selected of eager.selected) {
if (!selected.isActive)
await writeLocalFile(
path.join(liveDirectory, path.dirname(selected.sandboxName)),
path.basename(selected.sandboxName),
(await reload(selected.file.file_id)).bytes,
);
}
prepareQueuedCodeFileContext(context, [context], user);
const callback = createProvisionFilesCallback({
req,
fileStorage: deps.fileStorage,
agentToolContexts: new Map([['agent-a', context]]),
provisionToCodeEnv: service.provisionToCodeEnv,
provisionToVectorDB: service.provisionToVectorDB,
updateFile: async () => undefined,
updateCodeEnvRef: async () => undefined,
addEmbeddedEntity: async () => undefined,
});
const advertised = new Map(context.provisionState?.codeEnvDestinations);
let mounted: Awaited<ReturnType<typeof callback>>;
if (failOnceAt) {
await expect(callback(['execute_code'], 'agent-a')).rejects.toThrow();
expect(context.provisionState?.codeEnvDestinations).toEqual(advertised);
mounted = await callback(['execute_code'], 'agent-a');
expect(new Set(mounted.map((file) => file.name))).toEqual(new Set(advertised.values()));
} else mounted = await callback(['execute_code'], 'agent-a');
expect(getFiles).toHaveBeenCalledTimes(privateIds.length ? 1 : 0);
expect(checkSessionsAlive.mock.calls.length).toBeLessThanOrEqual(1);
const destinations = new Map(
eager.selected.map((selected) => [selected.file.file_id, selected.sandboxName]),
);
for (const [id, name] of advertised) destinations.set(id, name);
return { context, upload, advertised, mounted, destinations };
}
it.each([true, false])(
'initializes the newest snapshot before eager/lazy partitioning (deferred %s)',
async (deferred) => {
const first = await emit(Buffer.from('Version 1'), 'message-1', 'report.txt', 100);
await initializeCode([first.file_id], [], deferred);
expect((await reload(first.file_id)).file.metadata?.codeEnvRef).toBeDefined();
const second = await emit(Buffer.from('Version 2'), 'message-2', 'report.txt', 200);
const next = await initializeCode([first.file_id, second.file_id], [], deferred);
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
'Version 2',
);
expect(next.advertised.get(second.file_id)).toBe('report.txt');
expect(next.mounted.map((file) => file.name)).toEqual(['report.txt']);
expect(next.context.tool_resources?.execute_code?.files?.map((file) => file.file_id)).toEqual(
[second.file_id],
);
await initializeCode([first.file_id], [], deferred);
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
'Version 1',
);
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
},
);
it('keeps tenant authorization and private collisions through initialization and lazy setup', async () => {
const first = await emit(Buffer.from('Version 1'), 'message-1', 'report.txt', 100);
await initializeCode([first.file_id]);
const second = await emit(Buffer.from('Version 2'), 'message-2', 'report.txt', 200);
req.user.tenantId = 'tenant-b';
const foreign = await emit(Buffer.from('Foreign tenant'), 'foreign', 'report.txt', 300);
req.user.tenantId = 'tenant-a';
const privateFile = await emit(Buffer.from('Private setup'), 'private', 'report.txt', 400);
await methods.updateFile({ file_id: privateFile.file_id, context: FileContext.agents });
const next = await initializeCode(
[first.file_id, second.file_id, foreign.file_id],
[privateFile.file_id],
);
const name = next.advertised.get(second.file_id)!;
expect(name).toBe('report.txt');
expect((await readFile(path.join(directory, 'sandbox', name))).toString()).toBe('Version 2');
const privateName = next.advertised.get(privateFile.file_id)!;
expect(privateName).not.toBe(name);
expect((await readFile(path.join(directory, 'sandbox', privateName))).toString()).toBe(
'Private setup',
);
expect(next.advertised.has(foreign.file_id)).toBe(false);
});
it.each([
[true, true],
[true, false],
[false, true],
[false, false],
] as const)(
'keeps shared snapshots ahead of an already primed private resource (deferred %s, named %s)',
async (deferred, named) => {
const privateFile = await emit(Buffer.from('Private setup'), 'private', 'report.txt', 50);
await methods.updateFile({ file_id: privateFile.file_id, context: FileContext.agents });
await initializeCode([], [privateFile.file_id], deferred);
expect((await reload(privateFile.file_id)).file.metadata?.codeEnvRef).toMatchObject({
kind: 'agent',
id: 'agent-a',
sandboxFilename: 'report.txt',
});
if (!named)
await mongoose.models.File.updateOne(
{ file_id: privateFile.file_id },
{
$unset: {
'metadata.codeEnvRef.sandboxFilename': '',
'metadata.codeEnvRefs.default.sandboxFilename': '',
},
},
);
const first = await emit(Buffer.from('Shared V1'), 'shared-1', 'report.txt', 100);
const alone = await initializeCode([first.file_id], [], deferred);
const next = await initializeCode([first.file_id], [privateFile.file_id], deferred);
expect(next.destinations.get(first.file_id)).toBe(alone.destinations.get(first.file_id));
expect(next.destinations.get(first.file_id)).toBe('report.txt');
const privateName = next.destinations.get(privateFile.file_id)!;
expect(privateName).toMatch(/^report-.+\.txt$/);
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
'Shared V1',
);
expect((await readFile(path.join(directory, 'sandbox', privateName))).toString()).toBe(
'Private setup',
);
expect((await reload(privateFile.file_id)).file.metadata?.codeEnvRef?.sandboxFilename).toBe(
privateName,
);
const second = await emit(Buffer.from('Shared V2'), 'shared-2', 'report.txt', 200);
for (const ids of [
[first.file_id, second.file_id],
[first.file_id],
[first.file_id, second.file_id],
]) {
const restored = await initializeCode(ids, [privateFile.file_id], deferred);
expect(restored.destinations.get(privateFile.file_id)).toBe(privateName);
expect(restored.destinations.get(ids[ids.length - 1])).toBe('report.txt');
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
ids.length === 1 ? 'Shared V1' : 'Shared V2',
);
expect((await readFile(path.join(directory, 'sandbox', privateName))).toString()).toBe(
'Private setup',
);
}
},
);
it.each(['report.txt', 'nested/report-alias.txt'])(
'restores a rewritten upload at its original destination %s on old branches',
async (name) => {
const uploaded = await emit(Buffer.from('Uploaded original'), 'upload', 'upload.txt', 50);
await methods.updateFile({
file_id: uploaded.file_id,
context: FileContext.message_attachment,
metadata: {
codeDestination: true,
codeEnvRef: {
kind: 'user',
id: user,
storage_session_id: 'same-live-session',
file_id: 'same-live-object',
sandboxFilename: name,
},
},
});
await mongoose.models.File.updateOne(
{ file_id: uploaded.file_id },
{
$unset: { 'metadata.codeEnvSource': '', 'metadata.sourceDispatchedAt': '' },
$set: { createdAt: new Date(50) },
},
{ timestamps: false },
);
const live = path.join(directory, 'sandbox', name);
await writeLocalFile(path.dirname(live), path.basename(live), Buffer.from('Version 1'));
const first = await emit(await readFile(live), 'message-1', name, 100);
await initializeCode([uploaded.file_id, first.file_id]);
await writeLocalFile(path.dirname(live), path.basename(live), Buffer.from('Version 2'));
const second = await emit(await readFile(live), 'message-2', name, 200);
await initializeCode([uploaded.file_id, first.file_id, second.file_id]);
expect((await readFile(path.join(directory, 'sandbox', name))).toString()).toBe('Version 2');
const restored = await initializeCode([uploaded.file_id, first.file_id]);
expect((await readFile(path.join(directory, 'sandbox', name))).toString()).toBe('Version 1');
expect(
restored.context.tool_resources?.execute_code?.files?.map((file) => file.file_id),
).toEqual([first.file_id]);
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
},
);
it.each([true, false])(
'restores a literal WebP rewrite ahead of a stale converted legacy receipt (deferred %s)',
async (deferred) => {
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
.png()
.toBuffer();
const green = await sharp({
create: { width: 2, height: 2, channels: 3, background: 'green' },
})
.webp()
.toBuffer();
const legacy = await emit(blue, 'legacy', 'nested/chart.png', 100);
await methods.updateFile({ file_id: legacy.file_id, context: FileContext.execute_code });
await mongoose.models.File.updateOne(
{ file_id: legacy.file_id },
{ $unset: { 'metadata.codeEnvRef': '', 'metadata.codeEnvRefs': '' } },
);
const original = await initializeCode([legacy.file_id], [], deferred, undefined, true);
const name = original.destinations.get(legacy.file_id)!;
expect(name).toBe('nested/chart.webp');
expect((await reload(legacy.file_id)).file.metadata?.codeEnvRef?.sandboxFilename).toBe(name);
const live = path.join(directory, 'sandbox', name);
await writeLocalFile(path.dirname(live), path.basename(live), green);
const edited = await emit(await readFile(live), 'edited', name, 200);
const oldBytes = (await reload(legacy.file_id)).bytes;
const newBytes = (await reload(edited.file_id)).bytes;
expect(oldBytes).not.toEqual(newBytes);
let alias: string | undefined;
for (const ids of [
[legacy.file_id, edited.file_id],
[legacy.file_id, edited.file_id],
[legacy.file_id],
[legacy.file_id, edited.file_id],
]) {
await rm(path.join(directory, 'sandbox'), { recursive: true, force: true });
const restored = await initializeCode(ids, [], deferred, undefined, true);
expect(restored.destinations.size).toBe(ids.length);
if (ids.length === 1) {
expect(restored.destinations.get(legacy.file_id)).toBe(name);
expect(await readFile(live)).toEqual(oldBytes);
} else {
expect(restored.destinations.get(edited.file_id)).toBe(name);
const legacyName = restored.destinations.get(legacy.file_id)!;
expect(legacyName).toMatch(/^nested\/chart-.+\.webp$/);
if (alias) expect(legacyName).toBe(alias);
alias = legacyName;
expect(await readFile(live)).toEqual(newBytes);
expect(await readFile(path.join(directory, 'sandbox', legacyName))).toEqual(oldBytes);
}
expect((await reload(legacy.file_id)).bytes).toEqual(oldBytes);
expect((await reload(edited.file_id)).bytes).toEqual(newBytes);
}
},
);
it('retains distinct captured image paths across conversion, lazy mounts, eager reload and branch return', async () => {
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
.png()
.toBuffer();
const red = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'red' } })
.jpeg()
.toBuffer();
const first = await emit(blue, 'png', 'nested/chart.png', 100);
await initializeCode([first.file_id]);
const second = await emit(red, 'jpeg', 'nested/chart.jpeg', 200);
const next = await initializeCode([first.file_id, second.file_id]);
expect(next.destinations.size).toBe(2);
expect(new Set(next.destinations.values()).size).toBe(2);
const pngName = next.destinations.get(first.file_id)!;
const jpegName = next.destinations.get(second.file_id)!;
expect(pngName).toBe('nested/chart.webp');
expect(jpegName).toMatch(/^nested\/chart-.+\.webp$/);
const pngBytes = (await reload(first.file_id)).bytes;
const jpegBytes = (await reload(second.file_id)).bytes;
expect(await readFile(path.join(directory, 'sandbox', pngName))).toEqual(pngBytes);
expect(await readFile(path.join(directory, 'sandbox', jpegName))).toEqual(jpegBytes);
const eager = await initializeCode([first.file_id, second.file_id]);
expect(eager.destinations).toEqual(next.destinations);
const rewritten = await emit(red, 'png-rewritten', 'nested/chart.png', 300);
const newer = await initializeCode([first.file_id, second.file_id, rewritten.file_id]);
expect(newer.destinations.has(first.file_id)).toBe(false);
expect(newer.destinations.size).toBe(2);
await initializeCode([first.file_id, second.file_id]);
expect(await readFile(path.join(directory, 'sandbox', pngName))).toEqual(pngBytes);
expect(await readFile(path.join(directory, 'sandbox', jpegName))).toEqual(jpegBytes);
});
it.each([0, 1])(
'gives a new literal write ownership of assigned image mount %s without merging captures',
async (target) => {
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
.png()
.toBuffer();
const red = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'red' } })
.jpeg()
.toBuffer();
const green = await sharp({
create: { width: 2, height: 2, channels: 3, background: 'green' },
})
.webp()
.toBuffer();
const a = await emit(blue, 'png', 'nested/chart.png', 100);
await initializeCode([a.file_id]);
const b = await emit(red, 'jpeg', 'nested/chart.jpeg', 200);
const old = await initializeCode([a.file_id, b.file_id]);
const original = target === 0 ? a : b;
const editedPath = old.destinations.get(original.file_id)!;
const live = path.join(directory, 'sandbox', editedPath);
await writeLocalFile(path.dirname(live), path.basename(live), green);
const edited = await emit(await readFile(live), 'edited', editedPath, 300);
const next = await initializeCode(
[a.file_id, b.file_id, edited.file_id],
[],
true,
target === 1 ? editedPath : undefined,
);
expect(next.destinations.size).toBe(3);
expect(next.destinations.get(edited.file_id)).toBe(editedPath);
expect(next.destinations.get(original.file_id)).not.toBe(editedPath);
expect(await readFile(live)).toEqual((await reload(edited.file_id)).bytes);
expect(
await readFile(path.join(directory, 'sandbox', next.destinations.get(original.file_id)!)),
).toEqual((await reload(original.file_id)).bytes);
const again = await initializeCode([a.file_id, b.file_id, edited.file_id]);
expect(again.destinations).toEqual(next.destinations);
const branch = await initializeCode([a.file_id, b.file_id]);
expect(branch.destinations.size).toBe(2);
for (const file of [a, b])
expect(
await readFile(path.join(directory, 'sandbox', branch.destinations.get(file.file_id)!)),
).toEqual((await reload(file.file_id)).bytes);
const returned = await initializeCode([a.file_id, b.file_id, edited.file_id]);
expect(returned.destinations).toEqual(next.destinations);
expect(await readFile(live)).toEqual((await reload(edited.file_id)).bytes);
},
);
it.each(['report.txt', 'published/kept-name.txt'])(
'mounts the owned published V1 copy instead of mutable provenance at %s',
async (name) => {
const scope = {
userId: user,
tenantId: 'tenant-a',
conversationId: 'conversation-a',
runId: 'run-publish',
executionId: 'child',
agentId: 'agent-a',
sourceFileId: 'mutable-source',
};
const result = await processCodeOutput(
{
req,
id: scope.sourceFileId,
session_id: 'source-session',
name: 'report.txt',
conversationId: scope.conversationId,
messageId: 'published-message',
preparedBuffer: Buffer.from('Published V1'),
publication: {
scope,
provenance: {
runId: scope.runId,
executionId: scope.executionId,
agentId: scope.agentId,
sourceFileId: scope.sourceFileId,
publishedAt: new Date().toISOString(),
inputFileIds: [],
},
publish: methods.publishRunArtifactFile,
find: methods.findRunArtifactFile,
discard: async () => undefined,
},
},
deps,
);
if (!result?.file || !('file_id' in result.file)) throw new Error('Expected publication');
const published = result.file;
const unprovisioned = (await methods.getFiles({ file_id: published.file_id }))![0];
const probe = jest.fn(async () => ({ lastModified: new Date().toISOString() }));
const owned = {
kind: 'user' as const,
id: user,
storage_session_id: 'owned-session',
file_id: 'owned-copy',
sandboxFilename: name,
executionProfile: 'default' as const,
};
await withFileProvisioning({
file: published,
userId: user,
storage: deps.fileStorage,
destination: 'published-copy',
operation: async (_claim, onUploadStart) => {
onUploadStart();
await writeLocalFile(directory, 'owned-copy', (await reload(published.file_id)).bytes);
return {
value: owned,
commit: { code: { routeKey: 'default', ref: owned, legacyRef: owned } },
};
},
});
await writeLocalFile(directory, 'mutable-source', Buffer.from('Mutable V2'));
const record = (await methods.getFiles({ file_id: published.file_id }))![0];
const getFileInfo = jest.fn(async (ref: { file_id: string }) => ({
lastModified: new Date().toISOString(),
originalFilename: ref.file_id === 'owned-copy' ? name : 'wrong-live-name.txt',
}));
const selected = await selectCodeFiles({
files: [primingFile(record)],
routeKey: 'default',
getFileInfo,
});
expect(selected.selected).toHaveLength(1);
const mount = selected.selected[0];
expect(mount.isActive).toBe(true);
expect(await readFile(path.join(directory, mount.sourceRef.file_id!))).toEqual(
Buffer.from('Published V1'),
);
expect(mount.sourceRef).toMatchObject(owned);
expect(mount.sandboxName).toBe(name);
expect(getFileInfo).toHaveBeenCalledTimes(1);
expect(getFileInfo).toHaveBeenCalledWith(owned);
expect(record.context).toBe(FileContext.run_artifact);
expect(record.metadata?.codeEnvSource?.file_id).toBe('mutable-source');
expect(
(
await selectCodeFiles({
files: [primingFile(unprovisioned)],
routeKey: 'default',
getFileInfo: probe,
})
).selected,
).toEqual([]);
expect(probe).not.toHaveBeenCalled();
},
);
it.each(['no-storage-writer', 'processing-before-upload'] as const)(
'does not retain an unfinished snapshot claim after %s',
async (mode) => {
const kept = await emit(Buffer.from('Committed V1'), 'kept');
if (mode === 'no-storage-writer') deps.getStrategyFunctions = () => ({});
else
deps.determineFileType = async () => {
throw new Error('Processing failed before upload');
};
const attempt = processCodeOutput(
{
req,
id: 'fallback-source',
session_id: 'fallback-session',
name: 'report.txt',
messageId: 'fallback',
conversationId: 'conversation-a',
preparedBuffer: Buffer.from('Fallback bytes'),
},
deps,
);
if (mode === 'processing-before-upload')
await expect(attempt).rejects.toThrow('Processing failed before upload');
else expect((await attempt)?.file?.filepath).toBe('fallback');
expect((await reload(kept.file_id)).bytes.toString()).toBe('Committed V1');
expect(
await mongoose.models.File.countDocuments({
context: FileContext.execute_code_snapshot,
user,
tenantId: 'tenant-a',
}),
).toBe(1);
},
);
it('retains the unfinished claim after an uncertain external write', async () => {
const kept = await emit(Buffer.from('Committed V1'), 'kept');
const { saveBuffer } = deps.getStrategyFunctions(FileSources.local);
deps.getStrategyFunctions = () => ({
saveBuffer: async (input) => {
await saveBuffer!(input);
throw new Error('Storage acknowledgement lost');
},
});
await expect(emit(Buffer.from('Uncertain V2'), 'uncertain')).rejects.toThrow(
'File provisioning outcome is unknown',
);
const records = await mongoose.models.File.find({
context: FileContext.execute_code_snapshot,
user,
tenantId: 'tenant-a',
}).lean<IMongoFile[]>();
expect(records).toHaveLength(2);
const unfinished = records.find((file) => file.file_id !== kept.file_id)!;
expect(unfinished.filepath).toBeUndefined();
expect(unfinished.storageOperations?.provisioning).toHaveLength(1);
await expect(
deps.fileStorage.recovery.recover(
{ file_id: unfinished.file_id, userId: user, tenantId: 'tenant-a' },
deps.fileStorage,
),
).rejects.toThrow('File provisioning outcome is unknown');
expect(await mongoose.models.File.findOne({ file_id: unfinished.file_id }).lean()).toEqual(
unfinished,
);
expect((await reload(kept.file_id)).bytes.toString()).toBe('Committed V1');
});
it('retains V1 and V2 bytes and message references after same-path rewrites and reload', async () => {
const first = await emit(Buffer.from('Version 1'), 'message-1');
const second = await emit(Buffer.from('Version 2'), 'message-2');
const references = JSON.parse(JSON.stringify([first, second]));
expect((await reload(references[0].file_id)).bytes.toString()).toBe('Version 1');
expect((await reload(references[1].file_id)).bytes.toString()).toBe('Version 2');
expect(first.file_id).not.toBe(second.file_id);
expect((await reload(first.file_id)).file.messageId).toBe('message-1');
expect((await reload(second.file_id)).file.messageId).toBe('message-2');
expect(first.filename).toBe('report.txt');
expect(second.filename).toBe('report.txt');
});
it('keeps each image generation after conversion and reload', async () => {
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
.png()
.toBuffer();
const red = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'red' } })
.png()
.toBuffer();
const first = await emit(blue, 'image-1', 'nested/chart.png');
const second = await emit(red, 'image-2', 'nested/chart.png');
expect((await reload(first.file_id)).bytes).toEqual(
await sharp(blue).webp({ lossless: true }).toBuffer(),
);
expect((await reload(second.file_id)).bytes).toEqual(
await sharp(red).webp({ lossless: true }).toBuffer(),
);
expect(first.filepath).not.toBe(second.filepath);
});
it('isolates simultaneous same-name outputs, including identical reads and older dispatches', async () => {
const values = ['Version 1', 'Version 2', 'Version 2', 'Version 0'];
const files = await Promise.all(
values.map((value, index) =>
emit(Buffer.from(value), `message-${index}`, 'report.txt', 400 - index),
),
);
expect(new Set(files.map((file) => file.file_id)).size).toBe(values.length);
for (const [index, file] of files.entries()) {
expect((await reload(file.file_id)).bytes.toString()).toBe(values[index]);
expect((await reload(file.file_id)).file.user.toString()).toBe(user);
expect((await reload(file.file_id)).file.tenantId).toBe('tenant-a');
}
});
it.each([FileSources.s3, FileSources.cloudfront] as const)(
'keeps immutable tenant-owned object keys with %s storage',
async (source) => {
const client = new S3Client({ region: 'us-east-1' });
const remote = mockClient(client);
jest.spyOn(s3, 'initializeS3').mockReturnValue(client);
remote.on(PutObjectCommand).callsFake(async ({ Key, Body }) => {
const location = path.join(directory, 'objects', Key);
await writeLocalFile(path.dirname(location), path.basename(location), Body);
return {};
});
req.config.fileStrategy = source;
deps.getStrategyFunctions = () => ({
saveBuffer: (input) =>
saveBufferToS3({
...input,
urlBuilder: async (options) =>
`https://objects.example/${getS3Key({ ...options, basePath: options.basePath ?? 'images' })}`,
}),
});
try {
const first = await emit(Buffer.from('Version 1'), 'cloud-1');
const second = await emit(Buffer.from('Version 2'), 'cloud-2');
const original = await reload(first.file_id);
const rewritten = await reload(second.file_id);
expect(original.bytes.toString()).toBe('Version 1');
expect(rewritten.bytes.toString()).toBe('Version 2');
expect(original.file.storageKey).not.toBe(rewritten.file.storageKey);
expect(original.file.storageKey).toContain('tenant-a');
expect(original.file.storageKey).toContain(user);
expect(remote.commandCalls(PutObjectCommand)).toHaveLength(2);
} finally {
remote.restore();
client.destroy();
}
},
);
it('hydrates only message-referenced snapshots in the authenticated tenant and keeps the newest live path', async () => {
const first = await emit(Buffer.from('Version 1'), 'message-1', 'report.txt', 100);
const second = await emit(Buffer.from('Version 2'), 'message-2', 'report.txt', 200);
const late = await emit(Buffer.from('Version 0'), 'late-harvest', 'report.txt', 50);
const owner = { userId: user, tenantId: 'tenant-a' };
const files = await methods.getCodeGeneratedFiles(
'conversation-a',
[first.file_id, second.file_id, late.file_id],
owner,
);
expect(files).toHaveLength(3);
expect(
await methods.getCodeGeneratedFiles('conversation-a', [first.file_id], {
...owner,
tenantId: 'other-tenant',
}),
).toEqual([]);
expect(
await methods.getCodeGeneratedFiles('conversation-a', [first.file_id], {
...owner,
userId: new mongoose.Types.ObjectId().toString(),
}),
).toEqual([]);
expect(
(await methods.getCodeGeneratedFiles('conversation-a', [first.file_id], owner)).map(
(file) => file.file_id,
),
).toEqual([first.file_id]);
const getFileInfo = jest.fn(async () => ({
lastModified: new Date().toISOString(),
originalFilename: 'report.txt',
}));
const selected = await selectCodeFiles({
files: files.map((file) => ({
user: file.user.toString(),
file_id: file.file_id,
filename: file.filename,
filepath: file.filepath,
bytes: file.bytes,
type: file.type,
object: 'file',
embedded: false,
usage: file.usage,
context: FileContext.execute_code_snapshot,
createdAt: file.createdAt,
metadata: file.metadata,
})),
routeKey: 'default',
getFileInfo,
});
expect(selected.selected.map(({ file }) => file.file_id)).toEqual([second.file_id]);
expect(selected.selected[0].sandboxName).toBe('report.txt');
expect(selected.selected[0].isActive).toBe(false);
expect(getFileInfo).not.toHaveBeenCalled();
expect((await reload(first.file_id)).bytes.toString()).toBe('Version 1');
});
it('never reuses or deletes mutable recovery receipts after prime and a later rewrite', async () => {
const liveDirectory = path.join(directory, 'sandbox');
const livePath = path.join(liveDirectory, 'report.txt');
await writeLocalFile(liveDirectory, 'report.txt', Buffer.from('Version 1'));
const first = await emit(await readFile(livePath), 'message-1');
const ref = first.metadata!.codeEnvSource!;
await withFileProvisioning({
file: first,
userId: user,
storage: deps.fileStorage,
destination: 'code:default:report.txt',
operation: async (_claim, onUploadStart) => {
onUploadStart();
await writeLocalFile(liveDirectory, 'report.txt', (await reload(first.file_id)).bytes);
return { value: ref, commit: { code: { routeKey: 'default', ref, legacyRef: ref } } };
},
});
await writeLocalFile(liveDirectory, 'report.txt', Buffer.from('Version 2'));
const second = await emit(await readFile(livePath), 'message-2');
const original = (await reload(first.file_id)).file;
const getFileInfo = jest.fn(async () => ({ lastModified: new Date().toISOString() }));
const selected = await selectCodeFiles({
files: [
{
file_id: original.file_id,
filename: original.filename,
filepath: original.filepath,
bytes: original.bytes,
type: original.type,
metadata: original.metadata,
user: original.user.toString(),
object: 'file',
embedded: false,
usage: original.usage ?? 1,
context: FileContext.execute_code_snapshot,
},
],
routeKey: 'default',
getFileInfo,
});
expect(selected.selected).toHaveLength(1);
expect(selected.selected[0].isActive).toBe(false);
expect(selected.selected[0].ref).toBeUndefined();
expect(selected.selected[0].sandboxName).toBe('report.txt');
expect(getFileInfo).not.toHaveBeenCalled();
expect((await reload(first.file_id)).bytes.toString()).toBe('Version 1');
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
const httpClient = jest.fn(async () => undefined);
const remove = createCodeFileDeleter({
httpClient,
getAuthHeaders: async () => ({}),
getBaseUrl: () => 'https://code.example',
routeKey: () => 'default',
profileHeaders: () => ({}),
downloadQuery: () => '',
clearFileCodeEnvRefs: methods.clearFileCodeEnvRefs,
logger: { warn: () => undefined },
});
expect(original.metadata?.codeEnvRef).toMatchObject({ file_id: 'same-live-object' });
const removePrimary = createLocalFileDeleter({
deleteRagFile: async () => undefined,
logger: { warn: () => undefined, error: () => undefined },
});
const cleanupReq = {
user: req.user,
config: { fileConfig: { retainedFileCleanup: true } },
};
const removeSnapshot = createDeleteFileWithSecondaryStorage<
typeof cleanupReq,
typeof original,
never
>({
source: FileSources.local,
deleteFile: () => removePrimary({ user: req.user, config: { paths: paths() } }, original),
getDeleteFile: () => (_request, file) => remove(req, file),
clearFileEmbeddings: methods.clearFileEmbeddings,
isFileStorageClean: methods.isFileStorageClean,
fileStorage: { ...methods, recovery: deps.fileStorage.recovery },
logger: { warn: () => undefined },
});
await removeSnapshot(cleanupReq, original);
expect(httpClient).not.toHaveBeenCalled();
expect((await readFile(livePath)).toString()).toBe('Version 2');
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
const reconciled = await mongoose.models.File.findOne({
file_id: original.file_id,
}).lean<CodeFileCommitData>();
expect(reconciled?.metadata?.codeEnvRef).toBeUndefined();
expect(reconciled?.metadata?.codeEnvRefs?.default).toBeUndefined();
});
it('deletes one primary snapshot without deleting another version', async () => {
const first = await emit(Buffer.from('Version 1'), 'message-1');
const second = await emit(Buffer.from('Version 2'), 'message-2');
const remove = createLocalFileDeleter({
deleteRagFile: async () => undefined,
logger: { warn: () => undefined, error: () => undefined },
});
await remove({ user: req.user, config: { paths: paths() } }, first);
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
});
});

View file

@ -1,12 +1,6 @@
import path from 'path';
import { v4 } from 'uuid';
import {
megabyte,
fileConfig,
imageExtRegex,
inferMimeType,
mergeCodeEnvRef,
} from 'librechat-data-provider';
import { megabyte, fileConfig, imageExtRegex, inferMimeType } from 'librechat-data-provider';
import type { CodeEnvRef, FileSources, getEndpointFileConfig } from 'librechat-data-provider';
import type { CodeFileCommitData } from '@librechat/data-schemas';
import type {
@ -240,24 +234,8 @@ export async function processCodeOutput(
);
}
/**
* Ordinary outputs atomically reuse a file_id for this filename and conversation.
* Explicit publications instead receive a fresh attempt ID from the gateway.
*
* Claim by `safeName` (not raw `name`) so the claim and the eventual
* `createFile` agree on the filename column — otherwise weird inputs
* (e.g. `"proj name/file@v1.txt"`) would claim under the raw name and
* then write under the sanitized one, leaving the claim row orphaned.
*/
/**
* Dispatch-order stamp persisted with every write AND every claim insert
* (foreground writes dispatch ≈ now): the out-of-order guard below
* compares WRITER dispatch order, not wall-clock write time — an older
* task writing late must not make a newer task's harvest look stale, and
* a freshly claimed row must carry its claimant's stamp before the
* content write lands.
*/
const sourceDispatchedAt = freshClaimAfter ?? Date.now();
// Separate captures never share primary storage, even at the same sandbox path.
const sourceDispatchedAt = freshClaimAfter ?? currentDate.getTime();
const outputPersistence = createCodeOutputPersistence({
publication,
@ -272,36 +250,9 @@ export async function processCodeOutput(
user: req.user.id,
tenantId: req.user.tenantId,
sourceDispatchedAt,
snapshot: publication == null,
});
const file_id = claimed.file_id;
const isUpdate = file_id !== newFileId;
/**
* Out-of-order guard for detached (background) harvests: when the claimed
* row's last writer was dispatched AFTER this task (`freshClaimAfter` =
* this task's dispatch time), a newer run owns this filename slot. The
* `(filename, conversationId)` unique index means the stale bytes have
* nowhere else to live, so skip this file rather than overwrite fresh
* content — the harvest's stdout patch still lands, only the superseded
* attachment is omitted. Falls back to `updatedAt` for rows written
* before the stamp existed (the claim itself is timestamp-neutral).
*/
const lastWriterDispatchedAt =
claimed.metadata?.sourceDispatchedAt ??
(claimed.updatedAt != null ? new Date(claimed.updatedAt).getTime() : null);
if (isUpdate && freshClaimAfter != null && (lastWriterDispatchedAt ?? 0) > freshClaimAfter) {
logger.warn(
`[processCodeOutput] Skipping stale background output "${safeName}" (${file_id}): a newer run owns this filename`,
);
return null;
}
if (isUpdate) {
logger.debug(
`[processCodeOutput] Updating existing file "${safeName}" (${file_id}) instead of creating duplicate`,
);
}
/** Retain the completed output for token-qualified persistence after storage settles. */
let pendingOutput: CodeOutputStoredFile | undefined;
async function commitCodeFile(fileData: CodeOutputStoredFile): Promise<boolean> {
@ -312,29 +263,18 @@ export async function processCodeOutput(
const writeOutput = async (
onUploadStart: FileStorageUploadObserver,
): Promise<CodeOutputResult | null> => {
/**
* Preserve the original `messageId` on update. Each `processCodeOutput`
* call would otherwise overwrite it with the current run's run id, which
* decouples the file from the assistant message that originally created
* it. `getCodeGeneratedFiles` filters by `messageId IN <thread>`, so a
* stale id (e.g. from a later regeneration / failed re-read attempt)
* silently excludes the file from priming on subsequent turns.
*/
const persistedMessageId = isUpdate ? (claimed.messageId ?? messageId) : messageId;
/* A generated-output write replaces the file's bytes, so pointers to
* earlier content in another profile must not survive as reusable refs. */
const codeEnvReferenceSet = mergeCodeEnvRef(undefined, codeEnvRef);
// The sandbox object is a live path, not an owned immutable copy.
const codeEnvMetadata = {
...claimed.metadata,
...codeEnvReferenceSet,
codeEnvSource: codeEnvRef,
codeDestination: true,
sourceDispatchedAt,
};
if (isImage) {
const usage = isUpdate ? (claimed.usage ?? 0) + 1 : 1;
const usage = 1;
onUploadStart();
const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
const filepath = usage > 1 ? `${_file.filepath}?v=${Date.now()}` : _file.filepath;
const filepath = _file.filepath;
const storageMetadata = getStorageMetadata({
filepath: _file.filepath,
source: appConfig.fileStrategy,
@ -358,7 +298,7 @@ export async function processCodeOutput(
filepath,
...storageMetadata,
file_id,
messageId: persistedMessageId,
messageId,
usage,
filename: safeName,
conversationId,
@ -366,7 +306,7 @@ export async function processCodeOutput(
user: req.user.id,
tenantId: req.user.tenantId,
type: `image/${appConfig.imageOutputType}`,
createdAt: isUpdate ? claimed.createdAt : formattedDate,
createdAt: formattedDate,
updatedAt: formattedDate,
source: appConfig.fileStrategy,
context: outputPersistence.context,
@ -478,7 +418,7 @@ export async function processCodeOutput(
file_id,
filepath,
...storageMetadata,
messageId: persistedMessageId,
messageId,
object: 'file',
filename: safeName,
type: mimeType,
@ -490,8 +430,8 @@ export async function processCodeOutput(
metadata: codeEnvMetadata,
source: appConfig.fileStrategy,
context: outputPersistence.context,
usage: isUpdate ? (claimed.usage ?? 0) + 1 : 1,
createdAt: isUpdate ? claimed.createdAt : formattedDate,
usage: 1,
createdAt: formattedDate,
...(await retentionExpiryPromise),
};

View file

@ -110,7 +110,7 @@ describe('Code output publication gateway', () => {
expect(commit).toHaveBeenCalledWith(file);
const finalize = async () => file;
expect(persistence.finalize(file, finalize)).toBe(finalize);
expect(persistence.context).toBe(FileContext.execute_code);
expect(persistence.context).toBe(FileContext.execute_code_snapshot);
});
it('uses a unique attempt ID until publication and then returns the canonical file identity', async () => {

View file

@ -26,6 +26,7 @@ interface CodeOutputClaimInput {
user: string;
tenantId?: string | null;
sourceDispatchedAt?: number;
snapshot?: boolean;
}
interface CodeOutputClaim {
@ -103,7 +104,7 @@ export function createCodeOutputPersistence({
}): CodeOutputPersistence {
if (publication == null) {
return {
context: FileContext.execute_code,
context: FileContext.execute_code_snapshot,
claim,
commit,
trackStored: () => undefined,

View file

@ -81,6 +81,44 @@ describe('queued code-file destinations', () => {
expect(child.provisionState?.codeEnvDestinations?.get('shared')).toBe('data.csv');
});
it.each([false, true])(
'keeps one shared snapshot name across private contributors (reverse %s)',
(reverse) => {
const shared = file('shared', { context: FileContext.execute_code_snapshot });
const privateFile = file('private', {
metadata: {
codeEnvRef: {
kind: 'agent',
id: 'child',
storage_session_id: 'private-session',
file_id: 'private-object',
sandboxFilename: 'data.csv',
},
},
});
const parent = agent('parent', [shared]);
const child = agent('child', [shared, privateFile]);
child.provisionState!.agentScopedFileIds.add('private');
const contexts = reverse ? [child, parent] : [parent, child];
for (const context of contexts) prepareQueuedCodeFileContext(context, contexts, 'user-1');
for (const context of contexts) {
const uploads = planCodeFileUploads({
context,
contexts,
agentId: context.id,
userId: 'user-1',
useAdvertisedNames: true,
});
expect(uploads.find(({ file }) => file.file_id === 'shared')?.destination).toBe('data.csv');
expect(new Set(uploads.map(({ destination }) => destination)).size).toBe(uploads.length);
}
expect(child.provisionState?.codeEnvDestinations?.get('private')).toMatch(/^data-.+\.csv$/);
expect(child.dynamicToolContextMap?.queued_code_files).toContain(
child.provisionState?.codeEnvDestinations?.get('private'),
);
},
);
it('retains advertised paths after one file succeeds and another remains queued', () => {
const current = agent('parent', [file('first'), file('second')]);
prepareQueuedCodeFileContext(current, [current], 'user-1');

View file

@ -1,4 +1,8 @@
import { getCodeEnvRefForProfile, resolveSandboxFilename } from 'librechat-data-provider';
import {
FileContext,
getCodeEnvRefForProfile,
resolveSandboxFilename,
} from 'librechat-data-provider';
import type { TFile, TurnFileConsumers, AgentToolResources } from 'librechat-data-provider';
import type { CodeEnvFile } from '@librechat/agents';
import type { CodeExecutionRoute } from '../provision/service';
@ -9,6 +13,9 @@ import {
createCodeDestinationSet,
reserveCodeDestination,
sortCodeFilesByDestinationPriority,
selectCodeFileGenerations,
getCodeFileDestination,
planCodeSnapshotMounts,
} from './destinations';
import { appendCodeFileContextLine, getCodeFileContextLine, getCodeFileLocation } from './priming';
import { getCodeEnvUploadFilename } from './form';
@ -79,11 +86,13 @@ export function planCodeFileUploads({
}
const confirmedNames = new Set<string>();
const files = sortCodeFilesByDestinationPriority(
const files = selectCodeFileGenerations(
[...state.codeEnvFiles, ...liveFiles.filter((file) => !queuedIds.has(file.file_id))],
privateIds,
).filter((file): file is TFile => {
route,
).files.filter((file): file is TFile => {
if (!file) return false;
if (file.context === FileContext.execute_code_snapshot) return true;
const ref = getCodeEnvRefForProfile(file.metadata, route);
const recovery = state.codeEnvRecoveryNames?.get(file.file_id);
const entityId = state.agentScopedFileIds.has(file.file_id) ? agentId : undefined;
@ -99,17 +108,34 @@ export function planCodeFileUploads({
});
const destinations = createCodeDestinationSet();
const snapshotMounts = planCodeSnapshotMounts(
[...queued.values()].map((candidate) => candidate.file).concat(liveFiles),
privateIds,
route,
state.codeEnvDestinations,
);
for (const name of snapshotMounts.values()) reserveCodeDestination(destinations, name);
const liveNames = new Map<string, string>();
const reserveLiveFiles = (candidate: ProvisionToolContext): void => {
const candidateQueuedIds = new Set(
candidate.provisionState?.codeEnvFiles.map((file) => file.file_id),
);
for (const file of candidate.tool_resources?.execute_code?.files ?? []) {
if (candidateQueuedIds.has(file.file_id)) continue;
const live = candidate.tool_resources?.execute_code?.files ?? [];
const winners = new Set(
selectCodeFileGenerations(
[...(candidate.provisionState?.codeEnvFiles ?? []), ...live],
privateIds,
route,
).files.flatMap((file) => (file ? [file.file_id] : [])),
);
for (const file of live) {
if (candidateQueuedIds.has(file.file_id) || !winners.has(file.file_id)) continue;
const name =
getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
candidate.provisionState?.codeEnvDestinations?.get(file.file_id) ??
resolveSandboxFilename(file.filename, file.type);
file.context === FileContext.execute_code_snapshot
? getCodeFileDestination(file, route)
: (getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
candidate.provisionState?.codeEnvDestinations?.get(file.file_id) ??
resolveSandboxFilename(file.filename, file.type));
if (reserveCodeDestination(destinations, name) && !liveNames.has(file.file_id)) {
liveNames.set(file.file_id, name);
}
@ -131,8 +157,9 @@ export function planCodeFileUploads({
// An agent's frozen plan survives later discovery in other execution contexts.
for (const [id, name] of state.codeEnvDestinations ?? []) {
if (!queued.has(id)) continue;
reserveCodeDestination(destinations, name);
advertisedNames.set(id, name);
const destination = snapshotMounts.get(id) ?? name;
reserveCodeDestination(destinations, destination);
advertisedNames.set(id, destination);
}
// Inherited paths are hints until checked against this agent's live mounts.
for (const [id, candidate] of queued) {
@ -152,15 +179,18 @@ export function planCodeFileUploads({
if (!file) continue;
const candidate = queued.get(file.file_id);
const destination =
snapshotMounts.get(file.file_id) ??
advertisedNames.get(file.file_id) ??
liveNames.get(file.file_id) ??
claimCodeDestination(
destinations,
getCodeEnvUploadFilename(
resolveSandboxFilename(
getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
candidate?.recoveryName ??
file.filename,
file.context === FileContext.execute_code_snapshot
? getCodeFileDestination(file, route)
: (getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
candidate?.recoveryName ??
file.filename),
file.type,
),
),

View file

@ -0,0 +1,15 @@
import { hasToolResourceDestination } from 'librechat-data-provider';
import type { EToolResources, TFile } from 'librechat-data-provider';
/** Chooser declines remain declines; cleanup does not erase destination intent. */
export function allowsFileResource(
file: Partial<Pick<TFile, 'file_id' | 'metadata' | 'embedded'>>,
resource: EToolResources,
membership?: ReadonlyMap<EToolResources, ReadonlySet<string>>,
): boolean {
return (
file.metadata?.destinationChosen === false ||
membership?.get(resource)?.has(file.file_id ?? '') === true ||
hasToolResourceDestination(file, resource)
);
}

View file

@ -28,6 +28,7 @@ export enum FileContext {
agents = 'agents',
assistants = 'assistants',
execute_code = 'execute_code',
execute_code_snapshot = 'execute_code_snapshot',
image_generation = 'image_generation',
assistants_output = 'assistants_output',
message_attachment = 'message_attachment',
@ -230,6 +231,8 @@ export type TFile = {
*/
codeEnvRef?: CodeEnvRef;
codeEnvRefs?: CodeEnvRefMap;
/** Non-owning provenance. Never reused or deleted as a stored copy. */
codeEnvSource?: CodeEnvRef;
/** Dispatch-order stamp for the current source artifact generation. */
sourceDispatchedAt?: number;
/** Vector namespaces this file has been embedded into. */

View file

@ -451,6 +451,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
user: string;
tenantId?: string | null;
sourceDispatchedAt?: number;
snapshot?: boolean;
}) => Promise<IMongoFile>;
commitCodeFile: (data: CodeFileCommitData, sourceDispatchedAt?: number) => Promise<boolean>;
createFile: (data: Partial<IMongoFile>, disableTTL?: boolean) => Promise<IMongoFile | null>;
@ -908,7 +909,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
const filter = withOwnerScope(
{
file_id: { $in: fileIds },
context: { $ne: FileContext.execute_code },
context: { $nin: [FileContext.execute_code, FileContext.execute_code_snapshot] },
$or: orConditions,
},
ownerScope,
@ -982,7 +983,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
const filter = withOwnerScope(
{
conversationId,
context: FileContext.execute_code,
context: { $in: [FileContext.execute_code, FileContext.execute_code_snapshot] },
file_id: { $in: threadFileIds },
$or: [
{ 'metadata.codeEnvRef': { $exists: true } },
@ -1113,7 +1114,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
const filter = withOwnerScope(
{
file_id: { $in: fileIds },
context: { $ne: FileContext.execute_code },
context: { $nin: [FileContext.execute_code, FileContext.execute_code_snapshot] },
...(hydrateEverything ? {} : { $or: missingConditions }),
},
ownerScope,
@ -1154,7 +1155,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
const filter = withOwnerScope(
{
file_id: { $in: fileIds },
context: { $ne: FileContext.execute_code },
context: { $nin: [FileContext.execute_code, FileContext.execute_code_snapshot] },
$or: [
{ 'metadata.codeEnvRef': { $exists: true } },
{ 'metadata.codeEnvRefs': { $exists: true } },
@ -1190,6 +1191,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
* freshly claimed (not-yet-written) row still carries an ownership
* signal for the background harvest's stale-output guard. */
sourceDispatchedAt?: number;
snapshot?: boolean;
}): Promise<IMongoFile> {
const File = mongoose.models.File as Model<IMongoFile>;
const tenantFilter = data.tenantId ? { tenantId: data.tenantId } : { tenantId: null };
@ -1199,6 +1201,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
file_id: data.file_id,
user: data.user,
...(data.tenantId ? { tenantId: data.tenantId } : {}),
...(data.snapshot ? { expiresAt: new Date(Date.now() + 3600 * 1000) } : {}),
...(data.sourceDispatchedAt != null
? { metadata: { sourceDispatchedAt: data.sourceDispatchedAt } }
: {}),
@ -1209,7 +1212,8 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
{
filename: data.filename,
conversationId: data.conversationId,
context: FileContext.execute_code,
context: data.snapshot ? FileContext.execute_code_snapshot : FileContext.execute_code,
...(data.snapshot ? { file_id: data.file_id, user: data.user } : {}),
...tenantFilter,
...codeFileStorageFilter(),
},
@ -1251,7 +1255,9 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
delete fileData.expiresAt;
}
const generatedOutput = data.context === FileContext.execute_code;
const generatedOutput =
data.context === FileContext.execute_code ||
data.context === FileContext.execute_code_snapshot;
try {
return await File.findOneAndUpdate(
{

View file

@ -1,10 +1,12 @@
import mongoose from 'mongoose';
import { FileContext } from 'librechat-data-provider';
import { MongoMemoryServer } from 'mongodb-memory-server';
import type { FileStorageClaim, IMongoFile } from '~/types/file';
import type { AllMethods, FileStorageMethods } from './index';
import { _resetStrictCache } from '~/models/plugins/tenantIsolation';
import { tenantStorage, runAsSystem } from '~/config/tenantContext';
import { createFileStorageMethods } from './fileStorage';
import { createMethods } from './index';
import { createModels } from '~/models';
let File: mongoose.Model<IMongoFile>;
@ -71,6 +73,179 @@ describe('file storage fence', () => {
const publicMethods: Pick<AllMethods, keyof FileStorageMethods> = methods;
expect(Object.keys(publicMethods)).toHaveLength(5);
});
describe('unfinished snapshot releases', () => {
async function reserve() {
return createMethods(mongoose).claimCodeFile({
filename: 'report.txt',
conversationId: 'snapshot-conversation',
file_id: claim.file_id,
user: claim.userId,
tenantId: claim.tenantId,
sourceDispatchedAt: 100,
snapshot: true,
});
}
it('gives only a fresh snapshot reservation the existing upload TTL', async () => {
const startedAt = Date.now();
await reserve();
const reservation = await read();
expect(reservation?.expiresAt).toBeInstanceOf(Date);
expect(reservation!.expiresAt!.getTime()).toBeGreaterThanOrEqual(startedAt + 3_600_000);
expect(reservation!.expiresAt!.getTime()).toBeLessThanOrEqual(Date.now() + 3_600_000);
expect(reservation?.filepath).toBeUndefined();
expect(reservation?.storageOperations?.provisioning).toBeUndefined();
await File.init();
expect(await File.collection.indexes()).toContainEqual(
expect.objectContaining({ key: { expiresAt: 1 }, expireAfterSeconds: 3600 }),
);
});
it('retains the deadline when acquisition fails before the database write', async () => {
await reserve();
const reservation = await read();
expect(reservation?.expiresAt).toBeInstanceOf(Date);
jest.spyOn(File, 'updateOne').mockRejectedValueOnce(new Error('Database unavailable'));
await expect(peer.acquireFileProvision(claim)).rejects.toThrow('Database unavailable');
expect(await read()).toEqual(reservation);
expect((await read())?.storageOperations?.provisioning).toBeUndefined();
});
it('retains the deadline when another scope cannot acquire the reservation', async () => {
await reserve();
const reservation = await read();
expect(reservation?.expiresAt).toBeInstanceOf(Date);
expect(await peer.acquireFileProvision({ ...claim, tenantId: 'tenant-b' })).toBe(false);
expect(await read()).toEqual(reservation);
});
it('clears the deadline atomically even when the acquisition acknowledgement is lost', async () => {
await reserve();
expect((await read())?.expiresAt).toBeInstanceOf(Date);
jest.spyOn(peer, 'acquireFileProvision').mockImplementationOnce(async (data) => {
expect(await methods.acquireFileProvision(data)).toBe(true);
throw new Error('Acquisition acknowledgement lost');
});
await expect(peer.acquireFileProvision(claim)).rejects.toThrow(
'Acquisition acknowledgement lost',
);
const acquired = await read();
expect(acquired?.expiresAt).toBeUndefined();
expect(acquired?.storageOperations?.provisioning).toEqual([claim.token]);
});
it('does not extend an existing reservation deadline', async () => {
await reserve();
const deadline = new Date(Date.now() + 60_000);
await File.collection.updateOne(
{ file_id: claim.file_id },
{ $set: { expiresAt: deadline } },
);
const reservation = await read();
await reserve();
expect(await read()).toEqual(reservation);
});
it('does not add a deadline to an existing committed snapshot', async () => {
await reserve();
await methods.acquireFileProvision(claim);
await File.collection.updateOne(
{ file_id: claim.file_id },
{ $set: { filepath: '/uploads/committed', bytes: 100 } },
);
await methods.releaseFileProvision(claim);
const committed = await read();
expect(committed?.expiresAt).toBeUndefined();
await reserve();
expect(await read()).toEqual(committed);
});
it('leaves legacy claims without a provisional deadline', async () => {
const legacy = await createMethods(mongoose).claimCodeFile({
filename: 'legacy.txt',
conversationId: 'legacy-conversation',
file_id: 'legacy-file',
user: claim.userId,
tenantId: claim.tenantId,
});
expect(legacy.context).toBe(FileContext.execute_code);
expect(legacy.expiresAt).toBeUndefined();
});
it('deletes only the unfinished claim and accepts repeated releases', async () => {
await reserve();
expect((await read())?.object).toBe('file');
expect(await methods.acquireFileProvision(claim)).toBe(true);
expect(await methods.commitFileProvision({ claim, releaseOnly: true })).toBe(true);
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await read()).toBeNull();
expect(await peer.releaseFileProvision(claim)).toBe(true);
expect(await methods.acquireFileProvision(claim)).toBe(false);
expect(await methods.commitFileProvision({ claim, releaseOnly: true })).toBe(false);
});
it.each([
{ userId: new mongoose.Types.ObjectId().toString() },
{ tenantId: 'tenant-b' },
{ tenantId: null },
{ token: 'foreign' },
])('does not discard another scope or token: %j', async (overrides) => {
await reserve();
await methods.acquireFileProvision(claim);
const before = await read();
expect(await runAsSystem(() => peer.releaseFileProvision({ ...claim, ...overrides }))).toBe(
true,
);
expect(await read()).toEqual(before);
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await read()).toBeNull();
});
it('retains other claimants and deletes on the final concurrent release', async () => {
for (let round = 0; round < 4; round++) {
await reserve();
const claims = Array.from({ length: 16 }, (_, i) => ({ ...claim, token: `token-${i}` }));
expect(await Promise.all(claims.map(methods.acquireFileProvision))).toEqual(
claims.map(() => true),
);
expect(await methods.releaseFileProvision(claims[0])).toBe(true);
expect((await read())?.storageOperations?.provisioning).toHaveLength(15);
expect(await Promise.all(claims.slice(1).map(peer.releaseFileProvision))).toEqual(
claims.slice(1).map(() => true),
);
expect(await read()).toBeNull();
}
});
it.each([
{ filepath: '/uploads/committed' },
{ bytes: 0 },
{ storageKey: 'primary-object' },
{ context: FileContext.execute_code },
{ context: FileContext.message_attachment },
{ 'storageOperations.deletionPending': true },
{ 'storageOperations.deleting': 'deletion-owner' },
])('preserves committed content, other contexts and deletion intent: %j', async (fields) => {
await reserve();
await methods.acquireFileProvision(claim);
await File.collection.updateOne({ file_id: claim.file_id }, { $set: fields });
const before = await read();
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await read()).toEqual({
...before,
storageOperations: { ...before?.storageOperations, provisioning: [] },
});
});
it('retains committed snapshot metadata and primary content', async () => {
await seed({ context: FileContext.execute_code_snapshot });
const before = await read();
await methods.acquireFileProvision(claim);
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await read()).toEqual({ ...before, storageOperations: { provisioning: [] } });
});
});
it('retains every concurrent provisioning owner until release', async () => {
await seed();
const claims = ['code-a', 'code-b', 'search'].map((token) => ({ ...claim, token }));
@ -232,7 +407,7 @@ describe('file storage fence', () => {
it('never upserts missing files for any claim or release', async () => {
expect(await methods.acquireFileProvision(claim)).toBe(false);
expect(await methods.releaseFileProvision(claim)).toBe(false);
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await methods.acquireFileDeletion(deletion)).toBeNull();
expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(false);
expect(await File.collection.countDocuments()).toBe(0);
@ -250,7 +425,7 @@ describe('file storage fence', () => {
expect(await runAsSystem(() => peer.acquireFileProvision(other))).toBe(false);
expect(await runAsSystem(() => peer.acquireFileDeletion({ ...other, updatedAt }))).toBeNull();
expect(await methods.acquireFileProvision(claim)).toBe(true);
expect(await runAsSystem(() => peer.releaseFileProvision(other))).toBe(false);
expect(await runAsSystem(() => peer.releaseFileProvision(other))).toBe(true);
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
expect(
@ -289,7 +464,7 @@ describe('file storage fence', () => {
await tenantStorage.run(context, () => methods.acquireFileDeletion(deletion)),
).toBeNull();
expect(await methods.acquireFileProvision(claim)).toBe(true);
expect(await tenantStorage.run(context, () => methods.releaseFileProvision(claim))).toBe(false);
expect(await tenantStorage.run(context, () => methods.releaseFileProvision(claim))).toBe(true);
expect(await methods.releaseFileProvision(claim)).toBe(true);
expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
expect(

View file

@ -1,3 +1,4 @@
import { FileContext } from 'librechat-data-provider';
import type { FilterQuery, Model } from 'mongoose';
import type {
FileStorageClaim,
@ -50,15 +51,32 @@ export function createFileStorageMethods(mongoose: typeof import('mongoose')): {
return result.matchedCount === 1;
}
/** An already-applied release is successful; only the supplied token is removed. */
/** Safe releases also discard unfinished snapshots when the last claimant leaves. */
async function releaseFileProvision(data: FileStorageClaim): Promise<boolean> {
const File = mongoose.models.File as Model<IMongoFile>;
const result = await File.updateOne(
claimFilter(data),
{ $pull: { 'storageOperations.provisioning': data.token } },
{ timestamps: false },
);
return result.matchedCount === 1;
const scope = claimFilter(data);
const disposable: FilterQuery<IMongoFile> = {
context: FileContext.execute_code_snapshot,
filepath: { $exists: false },
bytes: { $exists: false },
storageKey: { $exists: false },
'storageOperations.provisioning': [data.token],
'storageOperations.deletionPending': { $ne: true },
'storageOperations.deleting': { $exists: false },
};
for (;;) {
const deleted = await File.deleteOne({ ...scope, ...disposable });
if (deleted.deletedCount === 1) return true;
const result = await File.updateOne(
// A concurrent release can leave us as the last claimant after the delete missed.
{ ...scope, $nor: [disposable] },
{ $pull: { 'storageOperations.provisioning': data.token } },
{ timestamps: false },
);
if (result.matchedCount === 1) return true;
// A lost delete acknowledgement must be safe to retry without recreating the row.
if ((await File.exists(scope)) == null) return true;
}
}
async function commitFileProvision(data: FileProvisionCommit): Promise<boolean> {

View file

@ -162,6 +162,10 @@ const file: Schema<IMongoFile> = new Schema(
type: codeEnvRefMapSchema,
default: undefined,
},
codeEnvSource: {
type: codeEnvRefSchema,
default: undefined,
},
/** Dispatch-order stamp of the last writer (or claimant, on insert):
* the background harvest's stale-output guard compares writer
* dispatch order so an older task settling late cannot overwrite a

View file

@ -221,6 +221,8 @@ export interface IMongoFile extends Omit<Document, 'model'> {
*/
codeEnvRef?: CodeEnvRef;
codeEnvRefs?: CodeEnvRefMap;
/** Non-owning provenance of an immutable generated snapshot. */
codeEnvSource?: CodeEnvRef;
/** Dispatch-order stamp for the current source artifact generation. */
sourceDispatchedAt?: number;
/** Vector namespaces this file has been embedded into. */