mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🦪 fix: Preserve Generated File Byte Snapshots (#18009)
* fix: Preserve Generated File Byte Snapshots * fix: Select Snapshot Generations Before Resource Partitioning * fix: Select Snapshot Destinations Before File Admission * fix: Separate Snapshot Capture Identity and File Consumers * fix: Separate Owned References and Literal Snapshot Mounts * fix: Clean Up Unfinished Snapshots and Normalize Upload Arbitration * fix: Keep Shared Snapshot Mounts Ahead of Private Receipts * test: Keep Snapshot Priority Regressions Type-Safe * fix: Preserve Literal Snapshot Paths During Legacy Recovery * fix: Reconcile Legacy Receipt Paths Before Recovery * fix: Reuse Fresh Unnamed Legacy Code References * fix: Expire Snapshot Reservations Before Provisioning --------- Co-authored-by: Lia <lia@librechat.ai> Co-authored-by: lia-by-librechat[bot] <328778573+lia-by-librechat[bot]@users.noreply.github.com>
This commit is contained in:
parent
413456941c
commit
c102e2d498
24 changed files with 2502 additions and 582 deletions
|
|
@ -427,7 +427,7 @@ describe('Code Process', () => {
|
|||
expect(result.file.file_id).toBe('existing-output');
|
||||
expect(result.file.messageId).toBe(baseParams.messageId);
|
||||
expect(mockCommitCodeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ messageId: 'original-message' }),
|
||||
expect.objectContaining({ messageId: baseParams.messageId }),
|
||||
undefined,
|
||||
);
|
||||
expect(mockCommitFileProvision).toHaveBeenCalledWith(
|
||||
|
|
@ -646,178 +646,37 @@ describe('Code Process', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('atomic file claim (via processCodeOutput)', () => {
|
||||
it('should reuse file_id from existing record via atomic claim', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-file-id',
|
||||
filename: 'test-file.txt',
|
||||
usage: 2,
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
});
|
||||
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(mockClaimCodeFile).toHaveBeenCalledWith({
|
||||
filename: 'test-file.txt',
|
||||
conversationId: 'conv-123',
|
||||
file_id: 'mock-uuid-1234',
|
||||
user: 'user-123',
|
||||
sourceDispatchedAt: expect.any(Number),
|
||||
});
|
||||
|
||||
expect(result.file_id).toBe('existing-file-id');
|
||||
expect(result.usage).toBe(3);
|
||||
expect(result.createdAt).toBe('2024-01-01T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('should create new file when no existing file found', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'mock-uuid-1234',
|
||||
user: 'user-123',
|
||||
});
|
||||
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(result.file_id).toBe('mock-uuid-1234');
|
||||
expect(result.usage).toBe(1);
|
||||
describe('snapshot reservation', () => {
|
||||
it('reserves a fresh capture rather than claiming the mutable filename slot', async () => {
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
const { file } = await processCodeOutput(baseParams);
|
||||
expect(mockClaimCodeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
filename: 'test-file.txt',
|
||||
conversationId: 'conv-123',
|
||||
file_id: 'mock-uuid-1234',
|
||||
user: 'user-123',
|
||||
snapshot: true,
|
||||
}),
|
||||
);
|
||||
expect(file.usage).toBe(1);
|
||||
expect(getRetentionExpiry).toHaveBeenCalledWith(baseParams.req);
|
||||
});
|
||||
|
||||
it('skips the file when the claim is newer than the background run (stale-harvest guard)', async () => {
|
||||
/* A newer run owns the filename slot and the (filename, conversationId)
|
||||
* unique index leaves stale bytes nowhere to live — the detached
|
||||
* harvest must not overwrite fresh content. */
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-file-id',
|
||||
filename: 'test-file.txt',
|
||||
messageId: 'newer-run-msg',
|
||||
updatedAt: '2024-01-02T00:00:00.000Z',
|
||||
});
|
||||
it('keeps an older background capture instead of discarding its history', async () => {
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
|
||||
const result = await processCodeOutput({
|
||||
...baseParams,
|
||||
freshClaimAfter: new Date('2024-01-01T00:00:00.000Z').getTime(),
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(mockCommitCodeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still reuses the claim when it predates the background run', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-file-id',
|
||||
filename: 'test-file.txt',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
});
|
||||
mockCommitCodeFile.mockResolvedValue(true);
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
|
||||
const { file: result } = await processCodeOutput({
|
||||
...baseParams,
|
||||
freshClaimAfter: new Date('2024-01-02T00:00:00.000Z').getTime(),
|
||||
});
|
||||
|
||||
expect(result.file_id).toBe('existing-file-id');
|
||||
const { file } = await processCodeOutput({ ...baseParams, freshClaimAfter: 100 });
|
||||
expect(file.metadata.sourceDispatchedAt).toBe(100);
|
||||
expect(mockCommitCodeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ file_id: 'existing-file-id' }),
|
||||
new Date('2024-01-02T00:00:00.000Z').getTime(),
|
||||
expect.objectContaining({ file_id: file.file_id }),
|
||||
100,
|
||||
);
|
||||
});
|
||||
|
||||
it('skips when a newer task holds an unwritten claim (insert stamp, no updatedAt yet)', async () => {
|
||||
/* A newer task claimed the filename but its content write is still in
|
||||
* flight: the claim-insert stamp alone must trip the guard. */
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-file-id',
|
||||
filename: 'test-file.txt',
|
||||
metadata: {
|
||||
sourceDispatchedAt: new Date('2024-01-02T00:00:00.000Z').getTime(),
|
||||
},
|
||||
});
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
|
||||
const result = await processCodeOutput({
|
||||
...baseParams,
|
||||
freshClaimAfter: new Date('2024-01-01T00:00:00.000Z').getTime(),
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(mockCommitCodeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('omits output when the commit boundary rejects an overtaken claim', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'mock-uuid-1234',
|
||||
user: 'user-123',
|
||||
});
|
||||
it('omits output when the storage commit is not confirmed', async () => {
|
||||
mockCommitCodeFile.mockResolvedValue(false);
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
|
||||
const result = await processCodeOutput({
|
||||
...baseParams,
|
||||
freshClaimAfter: new Date('2024-01-01T00:00:00.000Z').getTime(),
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(mockCommitCodeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
file_id: 'mock-uuid-1234',
|
||||
metadata: expect.objectContaining({
|
||||
sourceDispatchedAt: new Date('2024-01-01T00:00:00.000Z').getTime(),
|
||||
}),
|
||||
}),
|
||||
new Date('2024-01-01T00:00:00.000Z').getTime(),
|
||||
);
|
||||
});
|
||||
|
||||
it('commits when the conditional write matches (ownership held through the write)', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-file-id',
|
||||
filename: 'test-file.txt',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
});
|
||||
mockCommitCodeFile.mockResolvedValueOnce(true);
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
|
||||
const { file: result } = await processCodeOutput({
|
||||
...baseParams,
|
||||
freshClaimAfter: new Date('2024-01-02T00:00:00.000Z').getTime(),
|
||||
});
|
||||
|
||||
expect(result.file_id).toBe('existing-file-id');
|
||||
});
|
||||
|
||||
it('lets a newer task overwrite an OLDER task that wrote late (writer dispatch order wins)', async () => {
|
||||
/* Old task (dispatched Jan 1) settled late and wrote at Jan 3;
|
||||
* this task was dispatched Jan 2. Wall-clock updatedAt is newer
|
||||
* than our dispatch, but the WRITER is older — overwrite. */
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-file-id',
|
||||
filename: 'test-file.txt',
|
||||
updatedAt: '2024-01-03T00:00:00.000Z',
|
||||
metadata: {
|
||||
sourceDispatchedAt: new Date('2024-01-01T00:00:00.000Z').getTime(),
|
||||
},
|
||||
});
|
||||
mockCommitCodeFile.mockResolvedValue(true);
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
|
||||
|
||||
const { file: result } = await processCodeOutput({
|
||||
...baseParams,
|
||||
freshClaimAfter: new Date('2024-01-02T00:00:00.000Z').getTime(),
|
||||
});
|
||||
|
||||
expect(result.file_id).toBe('existing-file-id');
|
||||
expect(result.metadata.sourceDispatchedAt).toBe(
|
||||
new Date('2024-01-02T00:00:00.000Z').getTime(),
|
||||
);
|
||||
expect(await processCodeOutput({ ...baseParams, freshClaimAfter: 100 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -861,7 +720,7 @@ describe('Code Process', () => {
|
|||
'mock-uuid-1234.png',
|
||||
);
|
||||
expect(result.type).toBe('image/webp');
|
||||
expect(result.context).toBe(FileContext.execute_code);
|
||||
expect(result.context).toBe(FileContext.execute_code_snapshot);
|
||||
expect(result.filename).toBe('chart.png');
|
||||
});
|
||||
|
||||
|
|
@ -888,32 +747,12 @@ describe('Code Process', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should update existing image file with cache-busted filepath', async () => {
|
||||
const imageParams = { ...baseParams, name: 'chart.png' };
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-img-id',
|
||||
usage: 1,
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
});
|
||||
|
||||
const imageBuffer = Buffer.alloc(500);
|
||||
mockAxios.mockResolvedValue({ data: imageBuffer });
|
||||
convertImage.mockResolvedValue({ filepath: '/images/user-123/existing-img-id.webp' });
|
||||
|
||||
const { file: result } = await processCodeOutput(imageParams);
|
||||
|
||||
expect(convertImage).toHaveBeenCalledWith(
|
||||
mockReq,
|
||||
imageBuffer,
|
||||
'high',
|
||||
'existing-img-id.png',
|
||||
);
|
||||
expect(result.file_id).toBe('existing-img-id');
|
||||
expect(result.usage).toBe(2);
|
||||
expect(result.filepath).toMatch(/^\/images\/user-123\/existing-img-id\.webp\?v=\d+$/);
|
||||
expect(logger.debug).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Updating existing file'),
|
||||
);
|
||||
it('uses the snapshot object URL without mutable cache busting', async () => {
|
||||
mockAxios.mockResolvedValue({ data: Buffer.alloc(500) });
|
||||
convertImage.mockResolvedValue({ filepath: '/images/user-123/mock-uuid-1234.webp' });
|
||||
const { file } = await processCodeOutput({ ...baseParams, name: 'chart.png' });
|
||||
expect(file.filepath).toBe('/images/user-123/mock-uuid-1234.webp');
|
||||
expect(file.usage).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1353,68 +1192,22 @@ describe('Code Process', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('usage counter increment', () => {
|
||||
it('should set usage to 1 for new files', async () => {
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(result.usage).toBe(1);
|
||||
});
|
||||
|
||||
it('should increment usage for existing files', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-id',
|
||||
usage: 5,
|
||||
createdAt: '2024-01-01',
|
||||
});
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(result.usage).toBe(6);
|
||||
});
|
||||
|
||||
it('should handle existing file with undefined usage', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-id',
|
||||
createdAt: '2024-01-01',
|
||||
});
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(result.usage).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('metadata and file properties', () => {
|
||||
it('should include codeEnvRef in metadata with kind: user', async () => {
|
||||
it('should include codeEnvSource in metadata with kind: user', async () => {
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(result.metadata).toEqual({
|
||||
codeEnvRef: {
|
||||
codeEnvSource: {
|
||||
kind: 'user',
|
||||
id: 'user-123',
|
||||
storage_session_id: 'session-123',
|
||||
file_id: 'file-id-123',
|
||||
executionProfile: 'default',
|
||||
},
|
||||
codeEnvRefs: {
|
||||
default: {
|
||||
kind: 'user',
|
||||
id: 'user-123',
|
||||
storage_session_id: 'session-123',
|
||||
file_id: 'file-id-123',
|
||||
executionProfile: 'default',
|
||||
},
|
||||
},
|
||||
codeDestination: true,
|
||||
sourceDispatchedAt: expect.any(Number),
|
||||
});
|
||||
});
|
||||
|
|
@ -1430,10 +1223,10 @@ describe('Code Process', () => {
|
|||
executionRouteKey,
|
||||
});
|
||||
|
||||
expect(result.metadata.codeEnvRef).toEqual(
|
||||
expect(result.metadata.codeEnvSource).toEqual(
|
||||
expect.objectContaining({ executionProfile: 'stateful', executionRouteKey }),
|
||||
);
|
||||
expect(result.metadata.codeEnvRefs[executionRouteKey]).toEqual(result.metadata.codeEnvRef);
|
||||
expect(result.metadata.codeEnvRefs).toBeUndefined();
|
||||
});
|
||||
|
||||
/* Phase C lock-in: outputs are ALWAYS user-scoped, never skill-scoped.
|
||||
|
|
@ -1460,14 +1253,14 @@ describe('Code Process', () => {
|
|||
|
||||
// Each user's output ref is keyed by their own user id. The
|
||||
// `id` field tracks the requesting user, never the skill.
|
||||
expect(outputA.metadata.codeEnvRef).toEqual({
|
||||
expect(outputA.metadata.codeEnvSource).toEqual({
|
||||
kind: 'user',
|
||||
id: 'user-A',
|
||||
storage_session_id: 'session-123',
|
||||
file_id: 'file-id-123',
|
||||
executionProfile: 'default',
|
||||
});
|
||||
expect(outputB.metadata.codeEnvRef).toEqual({
|
||||
expect(outputB.metadata.codeEnvSource).toEqual({
|
||||
kind: 'user',
|
||||
id: 'user-B',
|
||||
storage_session_id: 'session-123',
|
||||
|
|
@ -1476,8 +1269,8 @@ describe('Code Process', () => {
|
|||
});
|
||||
|
||||
// No skill identity leaks into the output ref under any property.
|
||||
const refA = outputA.metadata.codeEnvRef;
|
||||
const refB = outputB.metadata.codeEnvRef;
|
||||
const refA = outputA.metadata.codeEnvSource;
|
||||
const refB = outputB.metadata.codeEnvSource;
|
||||
expect(refA.kind).not.toBe('skill');
|
||||
expect(refB.kind).not.toBe('skill');
|
||||
expect(refA).not.toHaveProperty('version');
|
||||
|
|
@ -1490,7 +1283,7 @@ describe('Code Process', () => {
|
|||
|
||||
const { file: result } = await processCodeOutput(baseParams);
|
||||
|
||||
expect(result.context).toBe(FileContext.execute_code);
|
||||
expect(result.context).toBe(FileContext.execute_code_snapshot);
|
||||
});
|
||||
|
||||
it('should include toolCallId and messageId in result', async () => {
|
||||
|
|
@ -1512,172 +1305,13 @@ describe('Code Process', () => {
|
|||
expect(mockCommitCodeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
file_id: 'mock-uuid-1234',
|
||||
context: FileContext.execute_code,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('persistedMessageId (regression for cross-turn priming)', () => {
|
||||
/**
|
||||
* `getCodeGeneratedFiles` filters by `messageId IN <thread message ids>`
|
||||
* to scope files to the current branch. If `processCodeOutput` overwrote
|
||||
* the file's `messageId` with the current run's id on every update, a
|
||||
* file re-touched by a later turn (e.g. a failed read attempt that
|
||||
* re-shells the same filename) would lose its link to the assistant
|
||||
* message that originally produced it. Subsequent turns then can't find
|
||||
* it via `getCodeGeneratedFiles`, the priming chain has nothing to seed,
|
||||
* and the model thinks its own prior-turn artifact disappeared.
|
||||
*
|
||||
* Contract:
|
||||
* - On UPDATE (claimCodeFile returned an existing record): the persisted
|
||||
* `messageId` is `claimed.messageId` (preserved). Falls back to the
|
||||
* current run's `messageId` when the existing record predates the
|
||||
* `messageId` field (legacy data).
|
||||
* - On CREATE (new file): the persisted `messageId` is the current run's.
|
||||
* - The runtime return value ALWAYS uses the current run's `messageId`
|
||||
* via `Object.assign(file, { messageId, toolCallId })` so the artifact
|
||||
* attaches to the correct tool_call in the live response.
|
||||
*/
|
||||
|
||||
/**
|
||||
* `processCodeOutput` mutates the file object after `commitCodeFile` returns
|
||||
* (`Object.assign(file, { messageId, toolCallId })`) so the runtime
|
||||
* caller sees the live messageId on the response. Reading
|
||||
* `mockCommitCodeFile.mock.calls[0][0]` directly would therefore reflect the
|
||||
* post-mutation state because JS captures by reference. To assert
|
||||
* what was actually PERSISTED, snapshot the args at call time.
|
||||
*/
|
||||
function snapshotCommitCodeFileArgs() {
|
||||
const snapshots = [];
|
||||
mockCommitCodeFile.mockImplementation(async (file) => {
|
||||
snapshots.push({ ...file });
|
||||
return true;
|
||||
});
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
it('preserves the original messageId in the persisted record on UPDATE', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-id',
|
||||
filename: 'sentinel.txt',
|
||||
usage: 1,
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
messageId: 'turn-1-original-msg',
|
||||
});
|
||||
const persisted = snapshotCommitCodeFileArgs();
|
||||
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
await processCodeOutput({
|
||||
...baseParams,
|
||||
name: 'sentinel.txt',
|
||||
messageId: 'turn-2-current-run-msg',
|
||||
});
|
||||
|
||||
expect(persisted[0].messageId).toBe('turn-1-original-msg');
|
||||
});
|
||||
|
||||
it('falls back to current run messageId on UPDATE when claimed.messageId is undefined (legacy record)', async () => {
|
||||
// Legacy record predates the persistedMessageId tracking.
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'legacy-id',
|
||||
filename: 'legacy.txt',
|
||||
usage: 1,
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
// messageId intentionally absent
|
||||
});
|
||||
const persisted = snapshotCommitCodeFileArgs();
|
||||
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
await processCodeOutput({
|
||||
...baseParams,
|
||||
name: 'legacy.txt',
|
||||
messageId: 'turn-N-current-run-msg',
|
||||
});
|
||||
|
||||
expect(persisted[0].messageId).toBe('turn-N-current-run-msg');
|
||||
});
|
||||
|
||||
it('uses the current run messageId on CREATE (no claimed record)', async () => {
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'mock-uuid-1234',
|
||||
user: 'user-123',
|
||||
});
|
||||
const persisted = snapshotCommitCodeFileArgs();
|
||||
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
await processCodeOutput({
|
||||
...baseParams,
|
||||
messageId: 'turn-1-create-msg',
|
||||
});
|
||||
|
||||
expect(persisted[0].messageId).toBe('turn-1-create-msg');
|
||||
});
|
||||
|
||||
it('returns the CURRENT run messageId in the runtime response even on UPDATE (artifact attribution)', async () => {
|
||||
// The persisted DB record keeps the original messageId, but the
|
||||
// returned object surfaces the live messageId so the artifact lands
|
||||
// on the correct tool_call in this run's response.
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-id',
|
||||
filename: 'sentinel.txt',
|
||||
usage: 1,
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
messageId: 'turn-1-original-msg',
|
||||
});
|
||||
const persisted = snapshotCommitCodeFileArgs();
|
||||
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
mockAxios.mockResolvedValue({ data: smallBuffer });
|
||||
|
||||
const { file: result } = await processCodeOutput({
|
||||
...baseParams,
|
||||
name: 'sentinel.txt',
|
||||
messageId: 'turn-2-current-run-msg',
|
||||
});
|
||||
|
||||
// DB preserves original
|
||||
expect(persisted[0].messageId).toBe('turn-1-original-msg');
|
||||
// Runtime return surfaces the live (current) messageId
|
||||
expect(result.messageId).toBe('turn-2-current-run-msg');
|
||||
});
|
||||
|
||||
it('preserves the original messageId on UPDATE for image files too', async () => {
|
||||
// Same contract as text files; the image branch builds its own file
|
||||
// record and would silently regress if the ternary diverged there.
|
||||
mockClaimCodeFile.mockResolvedValue({
|
||||
file_id: 'existing-img',
|
||||
filename: 'chart.png',
|
||||
usage: 1,
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
messageId: 'turn-1-image-msg',
|
||||
});
|
||||
const persisted = snapshotCommitCodeFileArgs();
|
||||
|
||||
const imageBuffer = Buffer.alloc(500);
|
||||
mockAxios.mockResolvedValue({ data: imageBuffer });
|
||||
convertImage.mockResolvedValue({
|
||||
filepath: '/uploads/chart.webp',
|
||||
bytes: 400,
|
||||
});
|
||||
|
||||
await processCodeOutput({
|
||||
...baseParams,
|
||||
name: 'chart.png',
|
||||
messageId: 'turn-2-current-img-msg',
|
||||
});
|
||||
|
||||
expect(persisted[0].messageId).toBe('turn-1-image-msg');
|
||||
});
|
||||
});
|
||||
|
||||
describe('socket pool isolation', () => {
|
||||
it('should pass dedicated keepAlive:false agents to axios for processCodeOutput', async () => {
|
||||
const smallBuffer = Buffer.alloc(100);
|
||||
|
|
@ -4799,6 +4433,63 @@ describe('Code Process', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('reuploads a displaced private receipt before mounting a shared snapshot', async () => {
|
||||
setupActiveSessions();
|
||||
const upload = jest.fn(async ({ filename }) => ({
|
||||
storage_session_id: `fresh-${filename}`,
|
||||
file_id: `fresh-${filename}`,
|
||||
filename,
|
||||
}));
|
||||
getStrategyFunctions.mockImplementation(() => ({
|
||||
getDownloadStream: jest.fn().mockResolvedValue('primary-bytes'),
|
||||
handleFileUpload: upload,
|
||||
}));
|
||||
const privateFile = codeFile({
|
||||
file_id: 'private',
|
||||
filename: 'report.txt',
|
||||
sandboxFilename: 'report.txt',
|
||||
storage_session_id: 'private-session',
|
||||
});
|
||||
privateFile.type = 'text/plain';
|
||||
privateFile.metadata.codeEnvRef.kind = 'agent';
|
||||
privateFile.metadata.codeEnvRef.id = 'agent-id';
|
||||
getFiles.mockResolvedValue([
|
||||
privateFile,
|
||||
{
|
||||
file_id: 'shared',
|
||||
user: 'user-123',
|
||||
filename: 'report.txt',
|
||||
type: 'text/plain',
|
||||
context: FileContext.execute_code_snapshot,
|
||||
source: 'local',
|
||||
filepath: '/uploads/shared',
|
||||
},
|
||||
]);
|
||||
const result = await prime({
|
||||
execute_code: { file_ids: ['private'], files: [{ file_id: 'shared' }] },
|
||||
});
|
||||
expect(result.files).toHaveLength(2);
|
||||
expect(result.files.find((file) => file.kind === 'user')).toMatchObject({
|
||||
name: 'report.txt',
|
||||
id: 'fresh-report.txt',
|
||||
});
|
||||
const privateMount = result.files.find((file) => file.kind === 'agent');
|
||||
expect(privateMount).toMatchObject({
|
||||
name: expect.stringMatching(/^report-.+\.txt$/),
|
||||
resource_id: 'agent-id',
|
||||
});
|
||||
expect(privateMount.id).toBe(`fresh-${privateMount.name}`);
|
||||
expect(upload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
filename: privateMount.name,
|
||||
stream: 'primary-bytes',
|
||||
kind: 'agent',
|
||||
id: 'agent-id',
|
||||
}),
|
||||
);
|
||||
expect(result.toolContext).toContain(privateMount.name);
|
||||
});
|
||||
|
||||
it('recovers a converted stale image even when its storage session has a fresh sibling', async () => {
|
||||
const upload = jest
|
||||
.fn()
|
||||
|
|
|
|||
|
|
@ -98,6 +98,8 @@ const mockOptionalChainWithEmptyCheck = jest.fn();
|
|||
const mockGetThreadData = jest.fn();
|
||||
|
||||
jest.mock('~/utils', () => ({
|
||||
createCodeApiRateLimitBudget: (...args: unknown[]) =>
|
||||
jest.requireActual('~/utils').createCodeApiRateLimitBudget(...args),
|
||||
extractLibreChatParams: (...args: unknown[]) => mockExtractLibreChatParams(...args),
|
||||
getModelMaxTokens: (...args: unknown[]) => mockGetModelMaxTokens(...args),
|
||||
optionalChainWithEmptyCheck: (...args: unknown[]) => mockOptionalChainWithEmptyCheck(...args),
|
||||
|
|
@ -139,6 +141,9 @@ jest.mock('../../middleware/modelBoundContent', () => {
|
|||
});
|
||||
|
||||
import { initializeAgent } from '../initialize';
|
||||
import { createProvisionFilesCallback } from '~/files/provision/callback';
|
||||
import { createFileProvisionRecovery } from '~/files/operations';
|
||||
import { selectCodeFiles } from '~/files/code/priming';
|
||||
import { primeResources } from '../resources';
|
||||
import { isFatalAgentInitializationError } from '../errors';
|
||||
import { canAgentGraphPause } from '../hitl/admission';
|
||||
|
|
@ -7202,3 +7207,383 @@ describe('initializeAgent — deferral without a loader registry', () => {
|
|||
expect(result.hasDeferredTools).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('snapshot destination admission invariants', () => {
|
||||
const MB = 1024 * 1024;
|
||||
const filterMock = jest.requireMock('~/files') as {
|
||||
filterFilesByEndpointRuntimeConfig: jest.Mock;
|
||||
};
|
||||
const realFilters = jest.requireActual<typeof import('~/files/filter')>('~/files/filter');
|
||||
const realResources = jest.requireActual<typeof import('../resources')>('../resources');
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
filterMock.filterFilesByEndpointRuntimeConfig
|
||||
.mockReset()
|
||||
.mockImplementation(realFilters.filterFilesByEndpointRuntimeConfig);
|
||||
jest.mocked(primeResources).mockReset().mockImplementation(realResources.primeResources);
|
||||
});
|
||||
afterEach(() => {
|
||||
filterMock.filterFilesByEndpointRuntimeConfig.mockReset().mockReturnValue([]);
|
||||
jest.mocked(primeResources).mockReset().mockResolvedValue({
|
||||
attachments: [],
|
||||
requestAttachments: [],
|
||||
agentContextAttachments: undefined,
|
||||
tool_resources: undefined,
|
||||
warnings: [],
|
||||
});
|
||||
});
|
||||
function snapshot(id: string, time: number, overrides: Partial<TFile> = {}): TFile {
|
||||
return {
|
||||
file_id: id,
|
||||
user: 'user-1',
|
||||
tenantId: 'tenant-1',
|
||||
filename: 'report.txt',
|
||||
filepath: `/uploads/${id}`,
|
||||
source: FileSources.local,
|
||||
type: 'text/plain',
|
||||
bytes: 12 * MB,
|
||||
object: 'file',
|
||||
embedded: false,
|
||||
usage: 0,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
createdAt: new Date(time),
|
||||
metadata: {
|
||||
codeDestination: true,
|
||||
sourceDispatchedAt: time,
|
||||
codeEnvSource: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
file_id: 'live-report',
|
||||
storage_session_id: 'sandbox',
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
async function run(
|
||||
files: TFile[],
|
||||
deferred: TFile[] = [],
|
||||
config: { fileSizeLimit?: number; supportedMimeTypes?: string[]; disabled?: boolean } = {},
|
||||
filters?: FiltersConfig,
|
||||
tools: string[] = [Tools.execute_code],
|
||||
) {
|
||||
const { agent, req, res, loadTools, db } = createMocks();
|
||||
agent.tools = tools;
|
||||
agent.endpoint = EModelEndpoint.openAI;
|
||||
req.user!.tenantId = 'tenant-1';
|
||||
req.config!.filters = filters;
|
||||
req.config!.fileConfig = {
|
||||
endpoints: { openAI: { totalSizeLimit: 20, fileSizeLimit: 20, ...config } },
|
||||
};
|
||||
mockExtractLibreChatParams.mockReturnValue({
|
||||
resendFiles: true,
|
||||
modelOptions: { model: agent.model },
|
||||
});
|
||||
const getFiles = jest.fn().mockResolvedValue([...files, ...deferred]);
|
||||
const result = await initializeAgent(
|
||||
{
|
||||
req,
|
||||
res,
|
||||
agent,
|
||||
loadTools,
|
||||
conversationId: 'snapshot-conversation',
|
||||
codeEnvAvailable: true,
|
||||
endpointOption: { endpoint: EModelEndpoint.agents },
|
||||
allowedProviders: new Set([Providers.OPENAI]),
|
||||
},
|
||||
{
|
||||
...db,
|
||||
getFiles,
|
||||
getConvoFiles: jest
|
||||
.fn()
|
||||
.mockResolvedValue([...files, ...deferred].map((file) => file.file_id)),
|
||||
getCodeGeneratedFiles: jest
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
files.filter((file) => file.context === FileContext.execute_code_snapshot),
|
||||
),
|
||||
getToolFilesByIds: jest
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
files.filter((file) => file.context !== FileContext.execute_code_snapshot),
|
||||
),
|
||||
getDeferredProvisionFiles: jest.fn().mockResolvedValue(deferred),
|
||||
},
|
||||
);
|
||||
const eager = await selectCodeFiles({
|
||||
files: result.tool_resources?.execute_code?.files ?? [],
|
||||
routeKey: 'default',
|
||||
getFileInfo: async () => ({ lastModified: new Date().toISOString() }),
|
||||
});
|
||||
const staged = new Map<string, string>();
|
||||
const callback = createProvisionFilesCallback({
|
||||
req,
|
||||
agentToolContexts: new Map([['agent-1', result]]),
|
||||
fileStorage: {
|
||||
recovery: createFileProvisionRecovery(),
|
||||
acquireFileProvision: jest.fn().mockResolvedValue(true),
|
||||
releaseFileProvision: jest.fn().mockResolvedValue(true),
|
||||
commitFileProvision: jest.fn().mockResolvedValue(true),
|
||||
},
|
||||
provisionToCodeEnv: async ({ file, sandboxFilename }) => {
|
||||
const name = sandboxFilename ?? file.filename;
|
||||
staged.set(name, file.file_id);
|
||||
return {
|
||||
referenceSet: {},
|
||||
sandboxFilename: name,
|
||||
refUpdate: {
|
||||
file_id: file.file_id,
|
||||
routeKey: 'default',
|
||||
ref: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
file_id: file.file_id,
|
||||
storage_session_id: 'sandbox',
|
||||
sandboxFilename: name,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
provisionToVectorDB: async ({ file }) => ({
|
||||
embedded: true,
|
||||
fileUpdate: { file_id: file.file_id, embedded: true },
|
||||
}),
|
||||
updateFile: async () => undefined,
|
||||
updateCodeEnvRef: async () => undefined,
|
||||
addEmbeddedEntity: async () => undefined,
|
||||
});
|
||||
await callback(tools, 'agent-1');
|
||||
expect(getFiles).toHaveBeenCalledTimes(1);
|
||||
expect(getFiles).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ user: 'user-1', tenantId: 'tenant-1' }),
|
||||
{},
|
||||
{},
|
||||
);
|
||||
return { result, staged, eager };
|
||||
}
|
||||
it.each([false, true])(
|
||||
'selects two 12 MiB versions before the 20 MiB budget (deferred newest %s)',
|
||||
async (deferred) => {
|
||||
const old = snapshot('v1', 100);
|
||||
const newest = snapshot('v2', 200);
|
||||
const { staged } = await run(deferred ? [old] : [old, newest], deferred ? [newest] : []);
|
||||
expect(staged.get('report.txt')).toBe('v2');
|
||||
expect([...staged.values()]).not.toContain('v1');
|
||||
},
|
||||
);
|
||||
it.each(['oversized', 'mime', 'disabled'] as const)(
|
||||
'never revives an older version when the newest is %s',
|
||||
async (policy) => {
|
||||
const overrides: Partial<TFile> = {};
|
||||
const config: Parameters<typeof run>[2] = {};
|
||||
if (policy === 'oversized') overrides.bytes = 25 * MB;
|
||||
if (policy === 'mime') {
|
||||
overrides.type = 'application/pdf';
|
||||
config.supportedMimeTypes = ['text/plain'];
|
||||
}
|
||||
if (policy === 'disabled') config.disabled = true;
|
||||
const { staged } = await run(
|
||||
[snapshot('v1', 100), snapshot('v2', 200, overrides)],
|
||||
[],
|
||||
config,
|
||||
);
|
||||
expect(staged.size).toBe(0);
|
||||
},
|
||||
);
|
||||
it('inspects the selected generation instead of reviving safe older bytes', async () => {
|
||||
await expect(
|
||||
run(
|
||||
[
|
||||
snapshot('v1', 100, { text: 'Safe old content' }),
|
||||
snapshot('v2', 200, { text: 'BLOCKED' }),
|
||||
],
|
||||
[],
|
||||
{},
|
||||
{
|
||||
files: {
|
||||
pii: {
|
||||
fields: ['extracted_text'],
|
||||
starterPatterns: [],
|
||||
customPatterns: [
|
||||
{ id: 'snapshot-policy', label: 'Snapshot policy', regex: 'BLOCKED' },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
it.each([
|
||||
{ tools: [Tools.execute_code, Tools.file_search] },
|
||||
{ tools: [Tools.execute_code] },
|
||||
{ tools: [Tools.file_search] },
|
||||
])('keeps independent consumer records when tools are $tools', async ({ tools }) => {
|
||||
const upload = snapshot('upload', 100, {
|
||||
context: FileContext.message_attachment,
|
||||
bytes: MB,
|
||||
embedded: true,
|
||||
metadata: { destinationChosen: true, codeDestination: true, searchDestination: true },
|
||||
});
|
||||
const output = snapshot('output', 200, { bytes: MB });
|
||||
const { result, staged } = await run([upload, output], [], {}, undefined, tools);
|
||||
expect(staged.get('report.txt')).toBe(
|
||||
tools.includes(Tools.execute_code) ? 'output' : undefined,
|
||||
);
|
||||
const search = result.tool_resources?.file_search?.files?.map((file) => file.file_id) ?? [];
|
||||
expect(search).toEqual(tools.includes(Tools.file_search) ? ['upload'] : []);
|
||||
});
|
||||
it.each([
|
||||
{ deferred: false, tools: [Tools.execute_code, Tools.file_search] },
|
||||
{ deferred: true, tools: [Tools.execute_code, Tools.file_search] },
|
||||
{ deferred: false, tools: [Tools.execute_code] },
|
||||
{ deferred: false, tools: [Tools.file_search] },
|
||||
])(
|
||||
'keeps newer search-only intent independent (deferred $deferred, tools $tools)',
|
||||
async ({ deferred, tools }) => {
|
||||
const output = snapshot('output', 100, { bytes: MB });
|
||||
const upload = snapshot('search', 200, {
|
||||
context: FileContext.message_attachment,
|
||||
bytes: MB,
|
||||
embedded: !deferred,
|
||||
metadata: { destinationChosen: true, searchDestination: true },
|
||||
});
|
||||
const { result, staged } = await run(
|
||||
deferred ? [output] : [output, upload],
|
||||
deferred ? [upload] : [],
|
||||
{},
|
||||
undefined,
|
||||
tools,
|
||||
);
|
||||
expect(staged.get('report.txt')).toBe(
|
||||
tools.includes(Tools.execute_code) ? 'output' : undefined,
|
||||
);
|
||||
expect(result.tool_resources?.file_search?.files?.map((file) => file.file_id) ?? []).toEqual(
|
||||
tools.includes(Tools.file_search) ? ['search'] : [],
|
||||
);
|
||||
},
|
||||
);
|
||||
it('retains model-bound input independently of its superseded code copy', async () => {
|
||||
const input = snapshot('model-input', 100, {
|
||||
context: FileContext.message_attachment,
|
||||
bytes: MB,
|
||||
llmDeliveryPath: 'text',
|
||||
text: 'Prompt input',
|
||||
metadata: { destinationChosen: true, codeDestination: true },
|
||||
});
|
||||
const { result, staged } = await run([input, snapshot('output', 200, { bytes: MB })]);
|
||||
expect(staged.get('report.txt')).toBe('output');
|
||||
expect(result.attachments?.map((file) => file?.file_id)).toContain('model-input');
|
||||
});
|
||||
it.each(['oversized', 'mime'] as const)(
|
||||
'keeps search input but never restores its old code bytes when newest is %s',
|
||||
async (policy) => {
|
||||
const upload = snapshot('upload', 100, {
|
||||
context: FileContext.message_attachment,
|
||||
bytes: MB,
|
||||
embedded: true,
|
||||
metadata: { destinationChosen: true, codeDestination: true, searchDestination: true },
|
||||
});
|
||||
const output = snapshot(
|
||||
'output',
|
||||
200,
|
||||
policy === 'oversized' ? { bytes: 25 * MB } : { bytes: MB, type: 'application/pdf' },
|
||||
);
|
||||
const { result, staged } = await run(
|
||||
[upload, output],
|
||||
[],
|
||||
policy === 'mime' ? { supportedMimeTypes: ['text/plain'] } : {},
|
||||
undefined,
|
||||
[Tools.execute_code, Tools.file_search],
|
||||
);
|
||||
expect(staged.size).toBe(0);
|
||||
expect(result.tool_resources?.file_search?.files?.map((file) => file.file_id)).toEqual([
|
||||
'upload',
|
||||
]);
|
||||
},
|
||||
);
|
||||
it('charges search-needed superseded inputs without reviving them for code when the union exceeds budget', async () => {
|
||||
const upload = snapshot('upload', 100, {
|
||||
context: FileContext.message_attachment,
|
||||
embedded: true,
|
||||
metadata: { destinationChosen: true, codeDestination: true, searchDestination: true },
|
||||
});
|
||||
const { result, staged } = await run([upload, snapshot('output', 200)], [], {}, undefined, [
|
||||
Tools.execute_code,
|
||||
Tools.file_search,
|
||||
]);
|
||||
expect(staged.size).toBe(0);
|
||||
expect(result.tool_resources?.file_search?.files?.map((file) => file.file_id)).toEqual([
|
||||
'upload',
|
||||
]);
|
||||
});
|
||||
it.each([
|
||||
[FileContext.execute_code_snapshot, true, false],
|
||||
[FileContext.execute_code, true, false],
|
||||
[FileContext.execute_code, false, false],
|
||||
[FileContext.execute_code, true, true],
|
||||
[FileContext.execute_code, false, true],
|
||||
] as const)(
|
||||
'charges independent capture and assigned-path writes separately across repeated initialization (%s, named %s, deferred newest %s)',
|
||||
async (context, named, deferred) => {
|
||||
const original = snapshot('png', 100, {
|
||||
context,
|
||||
filename: 'nested/chart.png',
|
||||
type: 'image/webp',
|
||||
metadata: {
|
||||
codeDestination: true,
|
||||
sourceDispatchedAt: 100,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
storage_session_id: 'sandbox',
|
||||
file_id: 'png-copy',
|
||||
sandboxFilename: named ? 'nested/chart.webp' : undefined,
|
||||
},
|
||||
},
|
||||
});
|
||||
const literal = snapshot('literal', 200, {
|
||||
filename: 'nested/chart.webp',
|
||||
type: 'image/webp',
|
||||
});
|
||||
const first = await run(
|
||||
deferred ? [original] : [original, literal],
|
||||
deferred ? [literal] : [],
|
||||
);
|
||||
expect([...first.staged.values()]).toEqual(['png']);
|
||||
expect(first.staged.has('nested/chart.webp')).toBe(false);
|
||||
expect(first.eager.selected.map((entry) => entry.sandboxName)).not.toContain(
|
||||
'nested/chart.webp',
|
||||
);
|
||||
const again = await run(
|
||||
deferred ? [original] : [original, literal],
|
||||
deferred ? [literal] : [],
|
||||
);
|
||||
expect(again.staged).toEqual(first.staged);
|
||||
},
|
||||
);
|
||||
it.each([false, true])(
|
||||
'normalizes an unreferenced converted upload before code admission (deferred newest %s)',
|
||||
async (deferred) => {
|
||||
const upload = snapshot('converted-upload', 100, {
|
||||
filename: 'chart.png',
|
||||
type: 'image/webp',
|
||||
context: FileContext.message_attachment,
|
||||
metadata: { destinationChosen: false, codeDestination: true },
|
||||
});
|
||||
const output = snapshot('literal', 200, { filename: 'chart.webp', type: 'image/webp' });
|
||||
const { staged } = await run(
|
||||
deferred ? [upload] : [upload, output],
|
||||
deferred ? [output] : [],
|
||||
);
|
||||
expect([...staged.entries()]).toEqual([['chart.webp', 'literal']]);
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps unrelated files subject to their original shared budget and order', async () => {
|
||||
const { staged } = await run([
|
||||
snapshot('first', 100, { filename: 'first.txt' }),
|
||||
snapshot('other', 200, { filename: 'other.txt', bytes: 9 * MB }),
|
||||
]);
|
||||
expect([...staged.entries()]).toEqual([['first.txt', 'first']]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ export function isToolOwnedAttachment(file: AttachmentTelemetryFile): boolean {
|
|||
| undefined;
|
||||
return (
|
||||
file.context === FileContext.execute_code ||
|
||||
file.context === FileContext.execute_code_snapshot ||
|
||||
file.embedded === true ||
|
||||
metadata?.searchDestination === true ||
|
||||
metadata?.codeDestination === true ||
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ import {
|
|||
} from '../projects/context';
|
||||
import { assertChatProjectInstructions, ChatProjectResourcesChangedError } from '../projects/turn';
|
||||
import { extractAgentContent, extractSkillContent } from '../protection/adapters/submissions';
|
||||
import { selectCodeFileGenerations, planCodeSnapshotMounts } from '~/files/code/destinations';
|
||||
import { createConfiguredContentInspector, inspectContent } from '../protection/runtime';
|
||||
import { assertAgentAttachmentLimits, isModelBoundAttachmentFile } from './attachments';
|
||||
import { assertModelBoundContent } from '../middleware/modelBoundContent';
|
||||
|
|
@ -147,6 +148,7 @@ import { resolveAgentCodeExecution } from '~/code/agent';
|
|||
import { hasActiveFileFieldPolicy } from '~/protection';
|
||||
import { applyBackgroundToolCalls } from './background';
|
||||
import { admitCurrentFileBudget } from './files/budget';
|
||||
import { allowsFileResource } from '~/files/consumers';
|
||||
import { applyDeferredToolLoading } from './deferral';
|
||||
import { applyTurnDelivery } from './files/delivery';
|
||||
import { generateArtifactsPrompt } from '~/prompts';
|
||||
|
|
@ -1923,13 +1925,45 @@ export async function initializeAgent(
|
|||
currentFiles = requestUsageFiles.concat(toolUsageFiles);
|
||||
}
|
||||
if (currentFiles?.length) {
|
||||
/* Before any check reads the route: endpoint filtering, model-bound limits and content
|
||||
* inspection below all have to judge each file by the route this turn delivers it by. */
|
||||
currentFiles = applyTurnDelivery(currentFiles, {
|
||||
routing: deliveryRouting,
|
||||
consumers: fileConsumers,
|
||||
});
|
||||
}
|
||||
const excludedCodeFileIds = new Set<string>();
|
||||
let codeSnapshotMounts: Map<string, string> | undefined;
|
||||
if (wantsCodeFiles) {
|
||||
const membership = new Map<EToolResources, ReadonlySet<string>>([
|
||||
[EToolResources.execute_code, new Set(agent.tool_resources?.execute_code?.file_ids)],
|
||||
[EToolResources.file_search, new Set(agent.tool_resources?.file_search?.file_ids)],
|
||||
]);
|
||||
const candidates = [...(currentFiles ?? []), ...deferredProvisionFiles].filter((file) =>
|
||||
allowsFileResource(file, EToolResources.execute_code, membership),
|
||||
);
|
||||
const privateIds = new Set(
|
||||
Object.values(agent.tool_resources ?? {}).flatMap((resource) => resource?.file_ids ?? []),
|
||||
);
|
||||
const selected = new Set(
|
||||
selectCodeFileGenerations(
|
||||
candidates,
|
||||
privateIds,
|
||||
codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
|
||||
).files.flatMap((file) => (file ? [file.file_id] : [])),
|
||||
);
|
||||
for (const file of candidates)
|
||||
if (!selected.has(file.file_id)) excludedCodeFileIds.add(file.file_id);
|
||||
codeSnapshotMounts = planCodeSnapshotMounts(
|
||||
candidates.filter((file) => selected.has(file.file_id)),
|
||||
privateIds,
|
||||
codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
|
||||
);
|
||||
const needed = (file: IMongoFile | TFile): boolean =>
|
||||
!excludedCodeFileIds.has(file.file_id) ||
|
||||
isModelBoundAttachmentFile(file) ||
|
||||
(wantsSearchFiles && allowsFileResource(file, EToolResources.file_search, membership));
|
||||
currentFiles = currentFiles?.filter(needed);
|
||||
deferredProvisionFiles = deferredProvisionFiles.filter(needed);
|
||||
}
|
||||
|
||||
let endpointFileType: EModelEndpoint | undefined;
|
||||
if (!paramEndpoints.has(agent.endpoint ?? '')) {
|
||||
|
|
@ -2097,6 +2131,8 @@ export async function initializeAgent(
|
|||
adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
|
||||
hasEntityEmbeddings: db.hasEntityEmbeddings,
|
||||
provisionCandidates: deferredProvisionFiles as unknown as TFile[],
|
||||
excludedCodeFileIds,
|
||||
codeSnapshotMounts,
|
||||
codeRouteKey: codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
|
||||
codeBaseUrl: codeExecutionContext.baseUrl,
|
||||
codeExecutionProfile: codeExecutionContext.executionProfile,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import {
|
|||
FileSources,
|
||||
getCodeEnvRefs,
|
||||
canToolResourceConsume,
|
||||
hasToolResourceDestination,
|
||||
} from 'librechat-data-provider';
|
||||
import type {
|
||||
AgentToolResources,
|
||||
|
|
@ -20,7 +19,12 @@ import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas';
|
|||
import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose';
|
||||
import type { CodeExecutionContext } from './execution';
|
||||
import type { ServerRequest } from '~/types';
|
||||
|
||||
import {
|
||||
getCodeFileDestination,
|
||||
selectCodeFileGenerations,
|
||||
planCodeSnapshotMounts,
|
||||
} from '~/files/code/destinations';
|
||||
import { allowsFileResource } from '~/files/consumers';
|
||||
import { getSafeErrorMetadata } from '~/utils/errors';
|
||||
import { TOOL_RESOURCE_KEYS } from './orphans';
|
||||
|
||||
|
|
@ -327,6 +331,7 @@ const categorizeFileForToolResources = ({
|
|||
requestFileSet,
|
||||
processedResourceFiles,
|
||||
agentScoped = false,
|
||||
codeFileIds,
|
||||
agentId,
|
||||
userId,
|
||||
}: {
|
||||
|
|
@ -336,12 +341,17 @@ const categorizeFileForToolResources = ({
|
|||
processedResourceFiles: Set<string>;
|
||||
/** Whether this file's vectors were embedded under the agent's entity_id. */
|
||||
agentScoped?: boolean;
|
||||
codeFileIds: ReadonlySet<string>;
|
||||
/** The agent whose vector namespace this turn will search, when one is scoped. */
|
||||
agentId?: string;
|
||||
/** Owner of the unscoped namespace, which is where anything not agent-scoped lands. */
|
||||
userId?: string;
|
||||
}): void => {
|
||||
if (file.metadata?.codeEnvRef || file.metadata?.codeEnvRefs) {
|
||||
if (
|
||||
file.context !== FileContext.execute_code_snapshot &&
|
||||
codeFileIds.has(file.file_id) &&
|
||||
(file.metadata?.codeEnvRef || file.metadata?.codeEnvRefs)
|
||||
) {
|
||||
addFileToResource({
|
||||
file,
|
||||
resourceType: EToolResources.execute_code,
|
||||
|
|
@ -443,6 +453,8 @@ const withDeferredCandidates = (
|
|||
const computeProvisionState = async ({
|
||||
req,
|
||||
attachments,
|
||||
codeFileIds,
|
||||
codeSnapshotMounts,
|
||||
resourcePrincipal,
|
||||
enabledToolResources,
|
||||
tool_resources,
|
||||
|
|
@ -463,6 +475,8 @@ const computeProvisionState = async ({
|
|||
}: {
|
||||
req?: ServerRequest;
|
||||
attachments: Array<TFile>;
|
||||
codeFileIds: ReadonlySet<string>;
|
||||
codeSnapshotMounts: ReadonlyMap<string, string>;
|
||||
agentId?: string;
|
||||
codeRouteKey?: string;
|
||||
/** Base URL of the deployment this turn runs on, used to probe its own refs. */
|
||||
|
|
@ -488,28 +502,8 @@ const computeProvisionState = async ({
|
|||
return undefined;
|
||||
}
|
||||
|
||||
/* The legacy chooser makes the destination an explicit user decision, so a file it
|
||||
* uploaded carries no reference for the destinations the user declined, and queueing on
|
||||
* a missing reference would read those declines as work to do. The decision belongs to
|
||||
* the file, not to this request: the endpoint deciding this turn need not be the one it
|
||||
* was uploaded under. Records predating the marker were written by the chooser flow,
|
||||
* so their existing resource membership/reference is the only durable destination evidence. */
|
||||
const cameFromChooser = (file: TFile): boolean => {
|
||||
const choice = file.metadata?.destinationChosen;
|
||||
return choice == null ? true : choice;
|
||||
};
|
||||
|
||||
/* Search intent survives vector cleanup without claiming the vectors still exist.
|
||||
* A chooser's other destinations remain declined unless membership or a copy proves them. */
|
||||
const allowsResource = (file: TFile, resourceType: EToolResources): boolean => {
|
||||
if (!cameFromChooser(file)) {
|
||||
return true;
|
||||
}
|
||||
const isPersistedResource =
|
||||
file.file_id != null &&
|
||||
persistedResourceMembership?.get(resourceType)?.has(file.file_id) === true;
|
||||
return isPersistedResource || hasToolResourceDestination(file, resourceType);
|
||||
};
|
||||
const allowsResource = (file: TFile, resource: EToolResources): boolean =>
|
||||
allowsFileResource(file, resource, persistedResourceMembership);
|
||||
const provisionable = attachments.filter(
|
||||
(file) =>
|
||||
file != null &&
|
||||
|
|
@ -603,7 +597,12 @@ const computeProvisionState = async ({
|
|||
const canProbeRoute = activeCodeRouteKey === 'default' || codeBaseUrl != null;
|
||||
const filesWithIdentifiers =
|
||||
needsCodeEnv && checkSessionsAlive && canProbeRoute
|
||||
? provisionable.filter((f) => f?.file_id && codeEnvRefForRoute(f, activeCodeRouteKey))
|
||||
? provisionable.filter(
|
||||
(f) =>
|
||||
f.context !== FileContext.execute_code_snapshot &&
|
||||
codeFileIds.has(f.file_id) &&
|
||||
codeEnvRefForRoute(f, activeCodeRouteKey),
|
||||
)
|
||||
: [];
|
||||
|
||||
/** Code API auth is optional: deployments may use a legacy key, JWT bearer minting,
|
||||
|
|
@ -654,6 +653,7 @@ const computeProvisionState = async ({
|
|||
const codeEnvFiles: TFile[] = [];
|
||||
const codeEnvRecoveryNames: NonNullable<ProvisionState['codeEnvRecoveryNames']> = new Map();
|
||||
const vectorDBFiles: TFile[] = [];
|
||||
const displacedCodeFileIds = new Set<string>();
|
||||
|
||||
for (const file of provisionable) {
|
||||
if (!file?.file_id) {
|
||||
|
|
@ -680,6 +680,7 @@ const computeProvisionState = async ({
|
|||
* the sandbox refuses aborts the tool batch over a file meant for the model. */
|
||||
if (
|
||||
needsCodeEnv &&
|
||||
codeFileIds.has(file.file_id) &&
|
||||
allowsResource(file, EToolResources.execute_code) &&
|
||||
canToolResourceConsume(EToolResources.execute_code, file.type ?? '')
|
||||
) {
|
||||
|
|
@ -693,7 +694,18 @@ const computeProvisionState = async ({
|
|||
* added to execute_code resources, so the check runs before the processed
|
||||
* guard. Clear both the legacy ref and its route entry, else getCodeEnvRefs
|
||||
* keeps resolving the dead session over the re-provisioned one. */
|
||||
if (isStale) {
|
||||
const plannedName = codeSnapshotMounts.get(file.file_id);
|
||||
const displaced =
|
||||
routeRef != null &&
|
||||
plannedName != null &&
|
||||
(plannedName !==
|
||||
(routeRef.sandboxFilename ?? getCodeFileDestination(file, activeCodeRouteKey)) ||
|
||||
(file.context === FileContext.execute_code &&
|
||||
plannedName !== getCodeFileDestination(file, activeCodeRouteKey)));
|
||||
if (file.context === FileContext.execute_code_snapshot || displaced) {
|
||||
codeEnvFiles.push(file);
|
||||
if (displaced) displacedCodeFileIds.add(file.file_id);
|
||||
} else if (isStale) {
|
||||
if (routeRef?.sandboxFilename) {
|
||||
codeEnvRecoveryNames.set(file.file_id, {
|
||||
name: routeRef.sandboxFilename,
|
||||
|
|
@ -749,12 +761,22 @@ const computeProvisionState = async ({
|
|||
}
|
||||
}
|
||||
|
||||
// A renamed receipt is not byte identity. Only its primary-storage reupload may mount it.
|
||||
const codeResource = tool_resources[EToolResources.execute_code];
|
||||
if (codeResource && displacedCodeFileIds.size > 0) {
|
||||
codeResource.file_ids = codeResource.file_ids?.filter((id) => !displacedCodeFileIds.has(id));
|
||||
codeResource.files = codeResource.files?.filter(
|
||||
(file) => !displacedCodeFileIds.has(file.file_id),
|
||||
);
|
||||
}
|
||||
|
||||
if (codeEnvFiles.length === 0 && vectorDBFiles.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
codeEnvFiles,
|
||||
codeEnvRecoveryNames,
|
||||
codeEnvDestinations: new Map(codeSnapshotMounts),
|
||||
vectorDBFiles,
|
||||
aliveFileIds: aliveFileIds ?? new Set(),
|
||||
agentScopedFileIds: new Set(scopedIds),
|
||||
|
|
@ -775,6 +797,8 @@ export const primeResources = async ({
|
|||
checkSessionsAlive,
|
||||
loadCodeApiKey,
|
||||
provisionCandidates,
|
||||
excludedCodeFileIds,
|
||||
codeSnapshotMounts: admittedSnapshotMounts,
|
||||
screenPersistentFiles,
|
||||
adoptLegacySearchEmbeddings,
|
||||
hasEntityEmbeddings,
|
||||
|
|
@ -802,6 +826,9 @@ export const primeResources = async ({
|
|||
* provisioning only and never returned as attachments: re-delivering an earlier
|
||||
* upload to the model on every later turn is not the intent. */
|
||||
provisionCandidates?: Array<TFile>;
|
||||
/** Pre-admission code losers may still be needed by search or prompt delivery. */
|
||||
excludedCodeFileIds?: ReadonlySet<string>;
|
||||
codeSnapshotMounts?: ReadonlyMap<string, string>;
|
||||
/** True when this endpoint still shows the explicit upload-destination chooser. */
|
||||
/** Applies the caller's endpoint and content policies. Persistent agent files are read
|
||||
* here rather than by the caller, so the caller has no chance to screen them itself
|
||||
|
|
@ -964,35 +991,77 @@ export const primeResources = async ({
|
|||
agentId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (screenPersistentFiles) {
|
||||
const admitted = screenPersistentFiles(persistedResourceFiles);
|
||||
/* The tool primers read tool_resources ids directly and re-check only access, so
|
||||
* a record dropped here still reaches the Code API or RAG unless its id goes with
|
||||
* it. Screening the hydrated list alone screens nothing. */
|
||||
if (admitted.length !== persistedResourceFiles.length) {
|
||||
const admittedIds = new Set(
|
||||
admitted.map((file) => file.file_id).filter((id): id is string => id != null),
|
||||
);
|
||||
const rejectedIds = persistedResourceFiles
|
||||
.map((file) => file.file_id)
|
||||
.filter((id): id is string => id != null && !admittedIds.has(id));
|
||||
const runtimeResources = [
|
||||
tool_resources[EToolResources.execute_code],
|
||||
tool_resources[EToolResources.file_search],
|
||||
];
|
||||
for (const resource of runtimeResources) {
|
||||
if (resource?.file_ids == null) {
|
||||
continue;
|
||||
}
|
||||
resource.file_ids = resource.file_ids.filter((id: string) => !rejectedIds.includes(id));
|
||||
}
|
||||
for (const id of rejectedIds) {
|
||||
toolResourceFileIds.delete(id);
|
||||
const files = _attachments ? await _attachments : [];
|
||||
const codeCandidates = withDeferredCandidates(
|
||||
persistedResourceFiles,
|
||||
withDeferredCandidates(
|
||||
files.filter((file): file is TFile => file != null),
|
||||
provisionCandidates,
|
||||
),
|
||||
);
|
||||
const eligibleCodeFiles = codeCandidates.filter(
|
||||
(file) =>
|
||||
!excludedCodeFileIds?.has(file.file_id) &&
|
||||
allowsFileResource(file, EToolResources.execute_code, persistedResourceMembership),
|
||||
);
|
||||
const codeFileIds = new Set(
|
||||
selectCodeFileGenerations(
|
||||
eligibleCodeFiles,
|
||||
persistedResourceFileIds,
|
||||
codeRouteKey,
|
||||
).files.flatMap((file) => (file ? [file.file_id] : [])),
|
||||
);
|
||||
const codeSnapshotMounts = planCodeSnapshotMounts(
|
||||
eligibleCodeFiles.filter((file) => codeFileIds.has(file.file_id)),
|
||||
persistedResourceFileIds,
|
||||
codeRouteKey,
|
||||
admittedSnapshotMounts,
|
||||
);
|
||||
const supersededCodeFileIds = new Set(
|
||||
codeCandidates.filter((file) => !codeFileIds.has(file.file_id)).map((file) => file.file_id),
|
||||
);
|
||||
const codeResource = tool_resources[EToolResources.execute_code];
|
||||
if (codeResource?.file_ids) {
|
||||
codeResource.file_ids = codeResource.file_ids.filter((id) => !supersededCodeFileIds.has(id));
|
||||
}
|
||||
|
||||
persistedResourceFiles = persistedResourceFiles.filter(
|
||||
(file) =>
|
||||
codeFileIds.has(file.file_id) ||
|
||||
(enabledToolResources?.has(EToolResources.file_search) &&
|
||||
allowsFileResource(file, EToolResources.file_search, persistedResourceMembership)) ||
|
||||
contextFileIds.has(file.file_id) ||
|
||||
imageEditFileIdSet.has(file.file_id),
|
||||
);
|
||||
if (screenPersistentFiles && persistedResourceFileIds.size > 0) {
|
||||
const admitted = screenPersistentFiles(persistedResourceFiles);
|
||||
/* The tool primers read tool_resources ids directly and re-check only access, so
|
||||
* a record dropped here still reaches the Code API or RAG unless its id goes with
|
||||
* it. Screening the hydrated list alone screens nothing. */
|
||||
if (admitted.length !== persistedResourceFiles.length) {
|
||||
const admittedIds = new Set(
|
||||
admitted.map((file) => file.file_id).filter((id): id is string => id != null),
|
||||
);
|
||||
const rejectedIds = persistedResourceFiles
|
||||
.map((file) => file.file_id)
|
||||
.filter((id): id is string => id != null && !admittedIds.has(id));
|
||||
const runtimeResources = [
|
||||
tool_resources[EToolResources.execute_code],
|
||||
tool_resources[EToolResources.file_search],
|
||||
];
|
||||
for (const resource of runtimeResources) {
|
||||
if (resource?.file_ids == null) {
|
||||
continue;
|
||||
}
|
||||
resource.file_ids = resource.file_ids.filter((id: string) => !rejectedIds.includes(id));
|
||||
}
|
||||
for (const id of rejectedIds) {
|
||||
toolResourceFileIds.delete(id);
|
||||
}
|
||||
persistedResourceFiles = admitted;
|
||||
}
|
||||
persistedResourceFiles = admitted;
|
||||
}
|
||||
|
||||
for (const file of persistedResourceFiles) {
|
||||
|
|
@ -1008,6 +1077,7 @@ export const primeResources = async ({
|
|||
tool_resources,
|
||||
requestFileSet,
|
||||
processedResourceFiles,
|
||||
codeFileIds,
|
||||
agentScoped: agentId != null && isAgentScopedFile(file),
|
||||
agentId,
|
||||
userId: resourcePrincipal?.id,
|
||||
|
|
@ -1042,6 +1112,8 @@ export const primeResources = async ({
|
|||
...(provisionCandidates ?? []),
|
||||
...resourceProvisionCandidates,
|
||||
]),
|
||||
codeFileIds,
|
||||
codeSnapshotMounts,
|
||||
resourcePrincipal,
|
||||
enabledToolResources,
|
||||
tool_resources,
|
||||
|
|
@ -1071,7 +1143,6 @@ export const primeResources = async ({
|
|||
};
|
||||
}
|
||||
|
||||
const files = await _attachments;
|
||||
const requestAttachmentFileIds = new Set<string>();
|
||||
|
||||
for (const file of files) {
|
||||
|
|
@ -1084,6 +1155,7 @@ export const primeResources = async ({
|
|||
tool_resources,
|
||||
requestFileSet,
|
||||
processedResourceFiles,
|
||||
codeFileIds,
|
||||
userId: resourcePrincipal?.id,
|
||||
});
|
||||
|
||||
|
|
@ -1111,6 +1183,8 @@ export const primeResources = async ({
|
|||
...(provisionCandidates ?? []),
|
||||
...resourceProvisionCandidates,
|
||||
]),
|
||||
codeFileIds,
|
||||
codeSnapshotMounts,
|
||||
resourcePrincipal,
|
||||
enabledToolResources,
|
||||
tool_resources,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { FileContext } from 'librechat-data-provider';
|
||||
import type { CodeEnvRef } from 'librechat-data-provider';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { createFileProvisionRecovery } from '../operations';
|
||||
|
|
@ -166,3 +167,27 @@ it.each([
|
|||
).rejects.toBe(error);
|
||||
expect(clearFileCodeEnvRefs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'reconciles snapshot receipts without deleting mutable paths (CAS %s)',
|
||||
async (matched) => {
|
||||
const { httpClient, clearFileCodeEnvRefs, remove } = setup();
|
||||
clearFileCodeEnvRefs.mockResolvedValue(matched);
|
||||
const file = {
|
||||
file_id: 'snapshot-v1',
|
||||
user: 'owner',
|
||||
tenantId: 'tenant-a',
|
||||
context: FileContext.execute_code_snapshot,
|
||||
metadata: { codeEnvRef: ref, codeEnvRefs: { default: ref } },
|
||||
};
|
||||
if (matched) await expect(remove(req, file)).resolves.toEqual({ deleted: true });
|
||||
else await expect(remove(req, file)).rejects.toThrow('Code storage reconciliation failed');
|
||||
expect(httpClient).not.toHaveBeenCalled();
|
||||
expect(clearFileCodeEnvRefs).toHaveBeenCalledWith({
|
||||
file_id: 'snapshot-v1',
|
||||
userId: 'owner',
|
||||
tenantId: 'tenant-a',
|
||||
refs: [{ routeKey: 'default', ref }],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { EModelEndpoint, getCodeEnvRefs } from 'librechat-data-provider';
|
||||
import { FileContext, EModelEndpoint, getCodeEnvRefs } from 'librechat-data-provider';
|
||||
import type { ClearFileCodeEnvRefsParams } from '@librechat/data-schemas';
|
||||
import type { CodeEnvReferenceSet } from 'librechat-data-provider';
|
||||
import type { AxiosRequestConfig } from 'axios';
|
||||
|
|
@ -39,6 +39,7 @@ export function createCodeFileDeleter({
|
|||
req: ServerRequest,
|
||||
file: {
|
||||
file_id?: string;
|
||||
context?: string;
|
||||
user?: { toString(): string };
|
||||
tenantId?: string;
|
||||
metadata?: CodeEnvReferenceSet;
|
||||
|
|
@ -47,6 +48,21 @@ export function createCodeFileDeleter({
|
|||
return async (req, file) => {
|
||||
const refs = getCodeEnvRefs(file.metadata);
|
||||
if (refs.length === 0) return undefined;
|
||||
// Recovery receipts name shared paths. Reconcile only this record, never the live object.
|
||||
if (file.context === FileContext.execute_code_snapshot) {
|
||||
if (
|
||||
!file.file_id ||
|
||||
!(await clearFileCodeEnvRefs({
|
||||
file_id: file.file_id,
|
||||
userId: file.user?.toString() ?? req.user?.id ?? '',
|
||||
tenantId: file.tenantId,
|
||||
refs: refs.map(([routeKey, ref]) => ({ routeKey, ref })),
|
||||
}))
|
||||
) {
|
||||
throw new Error('Code storage reconciliation failed');
|
||||
}
|
||||
return { deleted: true };
|
||||
}
|
||||
let skipped = false;
|
||||
for (const [executionRouteKey, ref] of refs) {
|
||||
const executionProfile = ref.executionProfile ?? 'default';
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
import { FileContext } from 'librechat-data-provider';
|
||||
import {
|
||||
claimCodeDestination,
|
||||
selectCodeFileGenerations,
|
||||
getCodeFileDestination,
|
||||
planCodeSnapshotMounts,
|
||||
reserveCodeDestination,
|
||||
createCodeDestinationSet,
|
||||
sortCodeFilesByDestinationPriority,
|
||||
|
|
@ -305,3 +309,105 @@ describe('sortCodeFilesByDestinationPriority', () => {
|
|||
expect(input).toEqual([older, newer]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoped generated destination selection', () => {
|
||||
const snapshot = (file_id: string, time: number, overrides = {}) => ({
|
||||
file_id,
|
||||
user: 'owner',
|
||||
tenantId: 'tenant-a',
|
||||
filename: 'nested/report.txt',
|
||||
type: 'text/plain',
|
||||
context: FileContext.execute_code_snapshot,
|
||||
metadata: { sourceDispatchedAt: time },
|
||||
...overrides,
|
||||
});
|
||||
it('arbitrates a rewritten upload alias at its real destination', () => {
|
||||
const upload = {
|
||||
...snapshot('upload', 50),
|
||||
context: FileContext.message_attachment,
|
||||
filename: 'display.txt',
|
||||
createdAt: new Date(50),
|
||||
metadata: {
|
||||
codeEnvRef: {
|
||||
kind: 'user' as const,
|
||||
id: 'owner',
|
||||
storage_session_id: 'session',
|
||||
file_id: 'object',
|
||||
sandboxFilename: 'nested/report.txt',
|
||||
},
|
||||
},
|
||||
};
|
||||
expect(
|
||||
selectCodeFileGenerations([upload, snapshot('v2', 200)]).files.map((file) => file?.file_id),
|
||||
).toEqual(['v2']);
|
||||
});
|
||||
it('never lets tenant or private generations supersede the shared owner generation', () => {
|
||||
const result = selectCodeFileGenerations(
|
||||
[
|
||||
snapshot('v1', 100),
|
||||
snapshot('v2', 200),
|
||||
snapshot('tenant-b', 300, { tenantId: 'tenant-b' }),
|
||||
snapshot('private', 400),
|
||||
],
|
||||
new Set(['private']),
|
||||
);
|
||||
expect(result.files.map((file) => file?.file_id)).toEqual(['tenant-b', 'v2', 'private']);
|
||||
});
|
||||
it('does not treat a mutable receipt alias as a snapshot identity', () => {
|
||||
const file = snapshot('v1', 100, {
|
||||
metadata: {
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'owner',
|
||||
storage_session_id: 'session',
|
||||
file_id: 'object',
|
||||
sandboxFilename: 'wrong-alias.txt',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(getCodeFileDestination(file)).toBe('nested/report.txt');
|
||||
});
|
||||
});
|
||||
|
||||
describe('legacy capture mount preferences', () => {
|
||||
it.each([
|
||||
['nested/chart.png', 'nested/chart.webp', 'image/webp', 'nested/chart.webp'],
|
||||
['report.txt', 'report-assigned.txt', 'text/plain', 'report.txt'],
|
||||
])(
|
||||
'preserves the turn plan but not persistent alias lineage for %s',
|
||||
(capture, name, type, canonical) => {
|
||||
const legacy = {
|
||||
file_id: 'legacy',
|
||||
filename: capture,
|
||||
type,
|
||||
context: FileContext.execute_code,
|
||||
metadata: {
|
||||
codeEnvRef: {
|
||||
kind: 'user' as const,
|
||||
id: 'owner',
|
||||
storage_session_id: 'session',
|
||||
file_id: 'object',
|
||||
sandboxFilename: name,
|
||||
},
|
||||
},
|
||||
};
|
||||
const literal = {
|
||||
file_id: 'literal',
|
||||
filename: name,
|
||||
type,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
};
|
||||
const admitted = planCodeSnapshotMounts([legacy, literal]);
|
||||
const alias = admitted.get(legacy.file_id)!;
|
||||
expect(admitted.get(literal.file_id)).toBe(name);
|
||||
expect(alias).not.toBe(name);
|
||||
expect(
|
||||
planCodeSnapshotMounts([legacy], undefined, 'default', admitted).get(legacy.file_id),
|
||||
).toBe(alias);
|
||||
legacy.metadata.codeEnvRef.sandboxFilename = alias;
|
||||
expect(planCodeSnapshotMounts([legacy]).get(legacy.file_id)).toBe(canonical);
|
||||
legacy.metadata.codeEnvRef.sandboxFilename = name;
|
||||
expect(planCodeSnapshotMounts([legacy]).get(legacy.file_id)).toBe(canonical);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,9 +1,16 @@
|
|||
import {
|
||||
FileContext,
|
||||
getCodeEnvRefForProfile,
|
||||
resolveSandboxFilename,
|
||||
} from 'librechat-data-provider';
|
||||
import type { CodeEnvReferenceSet } from 'librechat-data-provider';
|
||||
import {
|
||||
appendLeafSuffix,
|
||||
flattenArtifactPath,
|
||||
deterministicHexSuffix,
|
||||
FILENAME_SEGMENT_MAX_BYTES,
|
||||
} from '~/utils/files';
|
||||
import { getCodeEnvUploadFilename } from './form';
|
||||
|
||||
/**
|
||||
* Sandbox input files mount at a destination derived from their `name`, and
|
||||
|
|
@ -156,6 +163,7 @@ export function reserveCodeDestination(set: CodeDestinationSet, name: string): b
|
|||
}
|
||||
|
||||
interface CodeDestinationCandidate {
|
||||
context?: string;
|
||||
file_id?: string;
|
||||
createdAt?: Date | string | number;
|
||||
/** Last content write for a reused code-output record. `processCodeOutput`
|
||||
|
|
@ -174,6 +182,12 @@ function toTime(value: Date | string | number | undefined): number {
|
|||
}
|
||||
|
||||
function contentTime(file: CodeDestinationCandidate | null | undefined): number {
|
||||
if (
|
||||
file?.context === FileContext.execute_code_snapshot &&
|
||||
file.metadata?.sourceDispatchedAt != null
|
||||
) {
|
||||
return toTime(file.metadata.sourceDispatchedAt);
|
||||
}
|
||||
return Math.max(toTime(file?.metadata?.sourceDispatchedAt), toTime(file?.createdAt));
|
||||
}
|
||||
|
||||
|
|
@ -220,3 +234,154 @@ export function sortCodeFilesByDestinationPriority<T extends CodeDestinationCand
|
|||
return (a?.file_id ?? '').localeCompare(b?.file_id ?? '');
|
||||
});
|
||||
}
|
||||
|
||||
type CodePathCandidate = CodeDestinationCandidate & {
|
||||
filename?: string;
|
||||
tenantId?: string;
|
||||
user?: { toString(): string };
|
||||
type?: string;
|
||||
metadata?: (CodeEnvReferenceSet & { sourceDispatchedAt?: number }) | null;
|
||||
};
|
||||
|
||||
/** Mount naming only. Captured write paths, not converted names, identify versions. */
|
||||
export function getCodeFileDestination(file: CodePathCandidate, routeKey = 'default'): string {
|
||||
if (
|
||||
file.context !== FileContext.execute_code_snapshot &&
|
||||
file.context !== FileContext.execute_code
|
||||
) {
|
||||
const stored = getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename;
|
||||
if (stored != null) return stored;
|
||||
}
|
||||
const normalized = getCodeEnvUploadFilename(
|
||||
resolveSandboxFilename(file.filename ?? '', file.type ?? ''),
|
||||
);
|
||||
// A converted artifact's receipt is a mount-name hint only, never byte equivalence.
|
||||
if (file.context === FileContext.execute_code_snapshot && normalized !== file.filename) {
|
||||
return (
|
||||
getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename ?? normalized
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/** Select captured versions among code-eligible inputs before mount conversion. */
|
||||
export function selectCodeFileGenerations<T extends CodePathCandidate>(
|
||||
files: Array<T | null | undefined>,
|
||||
privateFileIds?: ReadonlySet<string>,
|
||||
routeKey = 'default',
|
||||
): { files: Array<T | null | undefined>; skippedSuperseded: number } {
|
||||
const destinations = new Map<string, T>();
|
||||
const generatedPaths = new Set<string>();
|
||||
let skippedSuperseded = 0;
|
||||
const ordered = sortCodeFilesByDestinationPriority(files, privateFileIds);
|
||||
const keyFor = (file: T): string => {
|
||||
const generated =
|
||||
file.context === FileContext.execute_code_snapshot ||
|
||||
file.context === FileContext.execute_code;
|
||||
const destination = generated
|
||||
? file.filename
|
||||
: (getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename ??
|
||||
getCodeEnvUploadFilename(resolveSandboxFilename(file.filename ?? '', file.type ?? '')));
|
||||
return JSON.stringify([
|
||||
privateFileIds?.has(file.file_id ?? '') ?? false,
|
||||
file.tenantId ?? null,
|
||||
file.user?.toString() ?? null,
|
||||
destination,
|
||||
]);
|
||||
};
|
||||
for (const file of ordered) {
|
||||
if (!file) continue;
|
||||
const key = keyFor(file);
|
||||
if (!destinations.has(key)) destinations.set(key, file);
|
||||
if (
|
||||
file.context === FileContext.execute_code_snapshot ||
|
||||
file.context === FileContext.execute_code
|
||||
)
|
||||
generatedPaths.add(key);
|
||||
}
|
||||
const selected = ordered.filter((file) => {
|
||||
if (!file) return true;
|
||||
const key = keyFor(file);
|
||||
// Distinct uploads remain independent unless a generated write owns this path.
|
||||
if (!generatedPaths.has(key) || destinations.get(key) === file) return true;
|
||||
skippedSuperseded++;
|
||||
return false;
|
||||
});
|
||||
return { files: selected, skippedSuperseded };
|
||||
}
|
||||
|
||||
/** Within each scope, literal snapshots precede legacy captures and converted snapshots. */
|
||||
export function planCodeSnapshotMounts<T extends CodePathCandidate>(
|
||||
files: Array<T | null | undefined>,
|
||||
privateFileIds?: ReadonlySet<string>,
|
||||
routeKey = 'default',
|
||||
preferred?: ReadonlyMap<string, string>,
|
||||
): Map<string, string> {
|
||||
const destinations = createCodeDestinationSet();
|
||||
const ordered = sortCodeFilesByDestinationPriority(files, privateFileIds);
|
||||
const snapshots: T[] = [];
|
||||
const receipts: T[] = [];
|
||||
const legacyCaptures: T[] = [];
|
||||
const isConverted = (file: T): boolean =>
|
||||
getCodeEnvUploadFilename(resolveSandboxFilename(file.filename ?? '', file.type ?? '')) !==
|
||||
file.filename;
|
||||
const seen = new Set<string>();
|
||||
for (const file of ordered) {
|
||||
if (!file || (file.file_id && seen.has(file.file_id))) continue;
|
||||
if (file.file_id) seen.add(file.file_id);
|
||||
if (file.context === FileContext.execute_code_snapshot) snapshots.push(file);
|
||||
else if (file.context === FileContext.execute_code) legacyCaptures.push(file);
|
||||
else receipts.push(file);
|
||||
}
|
||||
const priority = (file: T): number => {
|
||||
if (!isConverted(file)) return 0;
|
||||
const assigned =
|
||||
preferred?.has(file.file_id ?? '') ||
|
||||
getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey)?.sandboxFilename;
|
||||
return assigned ? 1 : 2;
|
||||
};
|
||||
snapshots.sort((a, b) => priority(a) - priority(b));
|
||||
const names = new Map<string, string>();
|
||||
for (const privateScope of [false, true]) {
|
||||
const inScope = (file: T): boolean =>
|
||||
(privateFileIds?.has(file.file_id ?? '') ?? false) === privateScope;
|
||||
for (const file of receipts) {
|
||||
if (!inScope(file)) continue;
|
||||
const ref = getCodeEnvRefForProfile(file.metadata ?? undefined, routeKey);
|
||||
const name =
|
||||
ref?.sandboxFilename ??
|
||||
(privateScope && ref != null ? getCodeFileDestination(file, routeKey) : undefined);
|
||||
if (!name) continue;
|
||||
if (!privateScope) {
|
||||
reserveCodeDestination(destinations, name);
|
||||
continue;
|
||||
}
|
||||
const destination = claimCodeDestination(destinations, name, file.file_id ?? name);
|
||||
if (file.file_id && destination !== name) names.set(file.file_id, destination);
|
||||
}
|
||||
const claimSnapshot = (file: T): void => {
|
||||
if (!file.file_id || !inScope(file)) return;
|
||||
names.set(
|
||||
file.file_id,
|
||||
claimCodeDestination(
|
||||
destinations,
|
||||
preferred?.get(file.file_id) ?? getCodeFileDestination(file, routeKey),
|
||||
file.file_id,
|
||||
),
|
||||
);
|
||||
};
|
||||
for (const file of snapshots) {
|
||||
if (!isConverted(file)) claimSnapshot(file);
|
||||
}
|
||||
for (const file of legacyCaptures) {
|
||||
if (!file.file_id || !inScope(file)) continue;
|
||||
// Retain this turn's admission plan, not an alias persisted on another branch.
|
||||
const name = preferred?.get(file.file_id) ?? getCodeFileDestination(file, routeKey);
|
||||
names.set(file.file_id, claimCodeDestination(destinations, name, file.file_id));
|
||||
}
|
||||
for (const file of snapshots) {
|
||||
if (isConverted(file)) claimSnapshot(file);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
const file = (id: string, name: string, ref: Partial<CodeEnvRef> = {}, time = 1): TFile =>
|
||||
({
|
||||
file_id: id,
|
||||
user: 'user',
|
||||
filename: name,
|
||||
type: 'text/csv',
|
||||
createdAt: new Date(time),
|
||||
|
|
@ -27,6 +28,127 @@ const file = (id: string, name: string, ref: Partial<CodeEnvRef> = {}, time = 1)
|
|||
const fresh = { lastModified: new Date().toISOString() };
|
||||
|
||||
describe('selectCodeFiles', () => {
|
||||
it.each(['default', 'stateful'] as const)(
|
||||
'reuses a fresh unnamed legacy ref with timestamp-only metadata on %s',
|
||||
async (routeKey) => {
|
||||
const legacy = {
|
||||
...file('legacy', 'rows.csv', { executionProfile: routeKey }),
|
||||
context: FileContext.execute_code,
|
||||
};
|
||||
const getFileInfo = jest.fn(async () => fresh);
|
||||
const result = await selectCodeFiles({ files: [legacy], routeKey, getFileInfo });
|
||||
|
||||
expect(result.selected).toHaveLength(1);
|
||||
expect(result.selected[0]).toMatchObject({
|
||||
ref: legacy.metadata!.codeEnvRef,
|
||||
sourceRef: legacy.metadata!.codeEnvRef,
|
||||
sandboxName: 'rows.csv',
|
||||
isActive: true,
|
||||
});
|
||||
expect(result.skippedNoRef).toBe(0);
|
||||
expect(result.skippedSuperseded).toBe(0);
|
||||
await expect(result.selected[0].getUploadTime()).resolves.toBe(fresh.lastModified);
|
||||
await expect(result.selected[0].getUploadTime()).resolves.toBe(fresh.lastModified);
|
||||
expect(getFileInfo).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps the newest snapshot at the live path across legacy records and route recovery', async () => {
|
||||
const legacy = {
|
||||
...file('legacy', 'rows.csv', { sandboxFilename: 'rows.csv' }),
|
||||
context: FileContext.execute_code,
|
||||
};
|
||||
const snapshot = {
|
||||
...file('snapshot', 'rows.csv', {}, 2),
|
||||
context: FileContext.execute_code_snapshot,
|
||||
};
|
||||
snapshot.metadata = { codeEnvSource: snapshot.metadata?.codeEnvRef, sourceDispatchedAt: 2 };
|
||||
const result = await selectCodeFiles({
|
||||
files: [legacy, snapshot],
|
||||
routeKey: 'stateful',
|
||||
getFileInfo: async () => fresh,
|
||||
});
|
||||
expect(result.selected.map(({ file }) => file.file_id)).toEqual(['snapshot']);
|
||||
expect(result.selected[0].sandboxName).toBe('rows.csv');
|
||||
expect(result.selected[0].ref).toBeUndefined();
|
||||
expect(result.skippedSuperseded).toBe(1);
|
||||
});
|
||||
|
||||
describe.each([
|
||||
{
|
||||
label: 'converted image',
|
||||
capture: 'nested/chart.png',
|
||||
name: 'nested/chart.webp',
|
||||
canonical: 'nested/chart.webp',
|
||||
type: 'image/webp',
|
||||
},
|
||||
{
|
||||
label: 'assigned text alias',
|
||||
capture: 'report.txt',
|
||||
name: 'report-assigned.txt',
|
||||
canonical: 'report.txt',
|
||||
type: 'text/plain',
|
||||
},
|
||||
])('$label', ({ capture, name, canonical, type }) => {
|
||||
it.each([
|
||||
['default', false],
|
||||
['default', true],
|
||||
['stateful', false],
|
||||
['stateful', true],
|
||||
] as const)(
|
||||
'keeps literal snapshot paths ahead of legacy receipts on %s (stale=%s)',
|
||||
async (routeKey, stale) => {
|
||||
const legacy = {
|
||||
...file('legacy', capture, {
|
||||
sandboxFilename: name,
|
||||
executionProfile: routeKey,
|
||||
}),
|
||||
type,
|
||||
context: FileContext.execute_code,
|
||||
};
|
||||
const snapshot = {
|
||||
...file('snapshot', name, {}, 2),
|
||||
type,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
};
|
||||
const result = await selectCodeFiles({
|
||||
files: [legacy, snapshot],
|
||||
routeKey,
|
||||
getFileInfo: async () => (stale ? null : fresh),
|
||||
});
|
||||
expect(result.selected).toHaveLength(2);
|
||||
expect(result.skippedSuperseded).toBe(0);
|
||||
expect(result.selected.find((entry) => entry.file.file_id === 'snapshot')).toMatchObject({
|
||||
sandboxName: name,
|
||||
isActive: false,
|
||||
});
|
||||
expect(result.selected.find((entry) => entry.file.file_id === 'legacy')).toMatchObject({
|
||||
sandboxName:
|
||||
canonical === name ? expect.stringMatching(/^nested\/chart-.+\.webp$/) : canonical,
|
||||
isActive: false,
|
||||
sourceRef: { file_id: 'legacy' },
|
||||
});
|
||||
const alias = result.selected.find((entry) => entry.file.file_id === 'legacy')!.sandboxName;
|
||||
legacy.metadata!.codeEnvRef!.sandboxFilename = alias;
|
||||
const repeated = await selectCodeFiles({
|
||||
files: [legacy, snapshot],
|
||||
routeKey,
|
||||
getFileInfo: async () => (stale ? null : fresh),
|
||||
});
|
||||
expect(repeated.selected.find((entry) => entry.file.file_id === 'legacy')).toMatchObject({
|
||||
sandboxName: alias,
|
||||
isActive: !stale && alias === canonical,
|
||||
});
|
||||
const oldBranch = await selectCodeFiles({
|
||||
files: [legacy],
|
||||
routeKey,
|
||||
getFileInfo: async () => (stale ? null : fresh),
|
||||
});
|
||||
expect(oldBranch.selected[0].sandboxName).toBe(canonical);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves provisioned aliases across turns and reuses legacy metadata probes', async () => {
|
||||
const getFileInfo = jest.fn(async (ref: CodeEnvRef) => ({
|
||||
...fresh,
|
||||
|
|
@ -125,10 +247,17 @@ describe('selectCodeFiles', () => {
|
|||
});
|
||||
expect(recovered.selected).toHaveLength(2);
|
||||
expect(new Set(recovered.selected.map((f) => f.sandboxName)).size).toBe(2);
|
||||
const restored = recovered.selected.map(({ file: original, sandboxName, sourceRef }) => ({
|
||||
...original,
|
||||
metadata: { codeEnvRef: { ...sourceRef, sandboxFilename: sandboxName } },
|
||||
}));
|
||||
const restored = recovered.selected.map(({ file: original, sandboxName, sourceRef }) => {
|
||||
if (!sourceRef.file_id || !sourceRef.storage_session_id)
|
||||
throw new Error('Expected owned legacy copy');
|
||||
const ref: CodeEnvRef = {
|
||||
...sourceRef,
|
||||
file_id: sourceRef.file_id,
|
||||
storage_session_id: sourceRef.storage_session_id,
|
||||
sandboxFilename: sandboxName,
|
||||
};
|
||||
return { ...original, metadata: { codeEnvRef: ref } };
|
||||
});
|
||||
const next = await selectCodeFiles({
|
||||
files: restored,
|
||||
routeKey: 'default',
|
||||
|
|
@ -205,6 +334,39 @@ describe('selectCodeFiles', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['default', true],
|
||||
['default', false],
|
||||
['stateful', true],
|
||||
['stateful', false],
|
||||
] as const)(
|
||||
'reuploads a displaced private receipt rather than trusting its live bytes on %s (named %s)',
|
||||
async (routeKey, named) => {
|
||||
const shared = { ...file('shared', 'rows.csv'), context: FileContext.execute_code_snapshot };
|
||||
const privateFile = file('private', 'rows.csv', {
|
||||
kind: 'agent',
|
||||
id: 'agent-a',
|
||||
sandboxFilename: named ? 'rows.csv' : undefined,
|
||||
executionProfile: routeKey,
|
||||
});
|
||||
const result = await selectCodeFiles({
|
||||
files: [privateFile, shared],
|
||||
privateFileIds: new Set(['private']),
|
||||
routeKey,
|
||||
getFileInfo: async () => ({ ...fresh, originalFilename: 'rows.csv' }),
|
||||
});
|
||||
expect(result.selected).toHaveLength(2);
|
||||
expect(result.selected.find((entry) => entry.file.file_id === 'shared')?.sandboxName).toBe(
|
||||
'rows.csv',
|
||||
);
|
||||
expect(result.selected.find((entry) => entry.file.file_id === 'private')).toMatchObject({
|
||||
sandboxName: expect.stringMatching(/^rows-.+\.csv$/),
|
||||
isActive: false,
|
||||
sourceRef: { kind: 'agent', id: 'agent-a', file_id: 'private' },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps the freshness decision paired with the selected image destination', async () => {
|
||||
const now = Date.parse('2026-09-10T12:00:00Z');
|
||||
const clock = jest.spyOn(Date, 'now').mockReturnValue(now);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ import {
|
|||
claimCodeDestination,
|
||||
createCodeDestinationSet,
|
||||
reserveCodeDestination,
|
||||
sortCodeFilesByDestinationPriority,
|
||||
selectCodeFileGenerations,
|
||||
getCodeFileDestination,
|
||||
planCodeSnapshotMounts,
|
||||
} from './destinations';
|
||||
import {
|
||||
codeServerHttpAgent,
|
||||
|
|
@ -92,10 +94,18 @@ export function checkCodeFileActive(dateString: string | undefined): boolean {
|
|||
return dateString != null && (Date.now() - new Date(dateString).getTime()) / 3_600_000 < 23;
|
||||
}
|
||||
|
||||
interface SnapshotCodeOwner {
|
||||
kind: 'user';
|
||||
id: string;
|
||||
storage_session_id?: undefined;
|
||||
file_id?: undefined;
|
||||
sandboxFilename?: undefined;
|
||||
}
|
||||
|
||||
interface PrimedCodeFile {
|
||||
file: TFile;
|
||||
ref: CodeEnvRef | undefined;
|
||||
sourceRef: CodeEnvRef;
|
||||
sourceRef: CodeEnvRef | SnapshotCodeOwner;
|
||||
sandboxName: string;
|
||||
isActive: boolean;
|
||||
getUploadTime: () => Promise<string | undefined>;
|
||||
|
|
@ -118,15 +128,28 @@ export async function selectCodeFiles({
|
|||
signal?: AbortSignal;
|
||||
}): Promise<{ selected: PrimedCodeFile[]; skippedNoRef: number; skippedSuperseded: number }> {
|
||||
let skippedNoRef = 0;
|
||||
let skippedSuperseded = 0;
|
||||
const generations = selectCodeFileGenerations(files, privateFileIds, routeKey);
|
||||
let skippedSuperseded = generations.skippedSuperseded;
|
||||
const snapshotMounts = planCodeSnapshotMounts(generations.files, privateFileIds, routeKey);
|
||||
const candidates = [];
|
||||
const registry = createCodeApiUploadRegistry();
|
||||
for (const file of sortCodeFilesByDestinationPriority(files, privateFileIds)) {
|
||||
for (const file of generations.files) {
|
||||
if (!file) {
|
||||
continue;
|
||||
}
|
||||
const ref = getCodeEnvRefForProfile(file.metadata, routeKey);
|
||||
const sourceRef = ref ?? getCodeEnvRefs(file.metadata)[0]?.[1];
|
||||
const snapshot = file.context === FileContext.execute_code_snapshot;
|
||||
const ownedRef = getCodeEnvRefForProfile(file.metadata, routeKey);
|
||||
// Snapshot receipts can name mounts, but never prove their current bytes.
|
||||
const ref = snapshot ? undefined : ownedRef;
|
||||
const sourceRef =
|
||||
ownedRef ??
|
||||
getCodeEnvRefs(file.metadata)[0]?.[1] ??
|
||||
(snapshot
|
||||
? {
|
||||
kind: 'user' as const,
|
||||
id: file.user.toString(),
|
||||
}
|
||||
: undefined);
|
||||
if (!sourceRef) {
|
||||
skippedNoRef++;
|
||||
continue;
|
||||
|
|
@ -138,7 +161,7 @@ export async function selectCodeFiles({
|
|||
scope: routeKey,
|
||||
concurrency,
|
||||
signal,
|
||||
task: () => getFileInfo(sourceRef),
|
||||
task: () => (ref ? getFileInfo(ref) : Promise.resolve(null)),
|
||||
}));
|
||||
candidates.push({ file, ref, sourceRef, getInfo });
|
||||
}
|
||||
|
|
@ -147,9 +170,19 @@ export async function selectCodeFiles({
|
|||
const { file, ref, sourceRef, getInfo } = candidate;
|
||||
const info = ref ? await getInfo() : null;
|
||||
signal?.throwIfAborted();
|
||||
const recovering = !ref || !checkCodeFileActive(info?.lastModified);
|
||||
const storedName = sourceRef.sandboxFilename ?? info?.originalFilename;
|
||||
const sandboxName = storedName ?? resolveSandboxFilename(file.filename, file.type);
|
||||
const plannedName = snapshotMounts.get(file.file_id);
|
||||
const recovering =
|
||||
!ref ||
|
||||
!checkCodeFileActive(info?.lastModified) ||
|
||||
(plannedName != null &&
|
||||
(plannedName !== (storedName ?? getCodeFileDestination(file, routeKey)) ||
|
||||
(file.context === FileContext.execute_code &&
|
||||
plannedName !== getCodeFileDestination(file, routeKey))));
|
||||
const sandboxName =
|
||||
file.context === FileContext.execute_code_snapshot
|
||||
? (snapshotMounts.get(file.file_id) ?? getCodeFileDestination(file, routeKey))
|
||||
: (plannedName ?? storedName ?? resolveSandboxFilename(file.filename, file.type));
|
||||
const destination = recovering
|
||||
? getCodeEnvUploadFilename(resolveSandboxFilename(sandboxName, file.type))
|
||||
: sandboxName;
|
||||
|
|
@ -287,7 +320,11 @@ export function getCodeFileContextLine(
|
|||
|
||||
/** Whether the code-file note mentions this file at `destination` at all. */
|
||||
export function isDescribedCodeFile(file: CodeContextFile, destination: string): boolean {
|
||||
return file.context !== FileContext.execute_code || destination !== file.filename;
|
||||
return (
|
||||
(file.context !== FileContext.execute_code &&
|
||||
file.context !== FileContext.execute_code_snapshot) ||
|
||||
destination !== file.filename
|
||||
);
|
||||
}
|
||||
|
||||
/** Provenance and preview state appended to a file shown under `shownName`. */
|
||||
|
|
@ -297,7 +334,7 @@ export function getCodeFileContextSuffix(
|
|||
shownName: string,
|
||||
): string {
|
||||
const origin =
|
||||
file.context === FileContext.execute_code
|
||||
file.context === FileContext.execute_code || file.context === FileContext.execute_code_snapshot
|
||||
? ` (written earlier as ${file.filename})`
|
||||
: `${agentScoped ? '' : ' (attached by user)'}${
|
||||
shownName !== file.filename ? ` (uploaded as ${file.filename})` : ''
|
||||
|
|
|
|||
990
packages/api/src/files/code/process.spec.ts
Normal file
990
packages/api/src/files/code/process.spec.ts
Normal file
|
|
@ -0,0 +1,990 @@
|
|||
import os from 'node:os';
|
||||
import sharp from 'sharp';
|
||||
import path from 'node:path';
|
||||
import mongoose from 'mongoose';
|
||||
import { Readable } from 'node:stream';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { mockClient } from 'aws-sdk-client-mock';
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { buffer as readStream } from 'node:stream/consumers';
|
||||
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { createMethods, fileSchema } from '@librechat/data-schemas';
|
||||
import {
|
||||
FileContext,
|
||||
FileSources,
|
||||
EToolResources,
|
||||
getEndpointFileConfig,
|
||||
mergeFileConfig,
|
||||
} from 'librechat-data-provider';
|
||||
import type { CodeFileCommitData, IMongoFile } from '@librechat/data-schemas';
|
||||
import type { TFile } from 'librechat-data-provider';
|
||||
import type { CodeOutputProcessorDeps } from './process';
|
||||
import type { TGetFiles } from '~/agents/resources';
|
||||
import type { CodeFileAgent } from './queued';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { hasOfficeHtmlPath, extractCodeArtifactText, getExtractedTextFormat } from './extract';
|
||||
import { createLocalFileDeleter, saveLocalBuffer, writeLocalFile } from '~/storage/local';
|
||||
import { createFileProvisionRecovery, withFileProvisioning } from '../operations';
|
||||
import { sanitizeArtifactPath, flattenArtifactPath } from '~/utils/files';
|
||||
import { createProvisionFilesCallback } from '../provision/callback';
|
||||
import { fromLocalUrlPath, stripCacheBust } from '~/storage/path';
|
||||
import { createDeleteFileWithSecondaryStorage } from '../storage';
|
||||
import { createProvisionService } from '../provision/service';
|
||||
import { saveBufferToS3, getS3Key } from '~/storage/s3/crud';
|
||||
import { createCodeApiUploadRegistry } from '~/utils/code';
|
||||
import { getStorageMetadata } from '~/storage/metadata';
|
||||
import { prepareQueuedCodeFileContext } from './queued';
|
||||
import { primeResources } from '~/agents/resources';
|
||||
import { classifyCodeArtifact } from './classify';
|
||||
import { createCodeFileDeleter } from './delete';
|
||||
import { processCodeOutput } from './process';
|
||||
import { selectCodeFiles } from './priming';
|
||||
import * as s3 from '~/cdn/s3';
|
||||
|
||||
/** Real Mongo claims, provisioning and filesystem bytes, including image conversion. */
|
||||
describe('Generated file snapshots', () => {
|
||||
let mongo: MongoMemoryServer;
|
||||
let directory: string;
|
||||
let methods: ReturnType<typeof createMethods>;
|
||||
let deps: CodeOutputProcessorDeps;
|
||||
let req: ServerRequest & Required<Pick<ServerRequest, 'user' | 'config'>>;
|
||||
const user = new mongoose.Types.ObjectId().toString();
|
||||
const paths = () => ({ publicPath: directory, uploads: path.join(directory, 'uploads') });
|
||||
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
|
||||
await mongoose.connect(mongo.getUri());
|
||||
const File = mongoose.model('File', fileSchema);
|
||||
await File.init();
|
||||
methods = createMethods(mongoose);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await mongoose.models.File.deleteMany({});
|
||||
directory = await mkdtemp(path.join(os.tmpdir(), 'generated-snapshots-'));
|
||||
req = {
|
||||
user: { id: user, tenantId: 'tenant-a' },
|
||||
app: { locals: { codeApiUploadRegistry: createCodeApiUploadRegistry() } },
|
||||
config: { fileStrategy: FileSources.local, imageOutputType: 'webp' },
|
||||
} as unknown as typeof req;
|
||||
const saveBuffer: NonNullable<
|
||||
ReturnType<CodeOutputProcessorDeps['getStrategyFunctions']>['saveBuffer']
|
||||
> = (input) => saveLocalBuffer({ ...input, paths: paths(), basePath: 'uploads' });
|
||||
deps = {
|
||||
fileStorage: { ...methods, recovery: createFileProvisionRecovery() },
|
||||
claimCodeFile: methods.claimCodeFile,
|
||||
commitCodeFile: methods.commitCodeFile,
|
||||
getCodeOutputFileSettings: () => ({
|
||||
endpointFileConfig: getEndpointFileConfig({
|
||||
fileConfig: mergeFileConfig({}),
|
||||
endpoint: 'agents',
|
||||
}),
|
||||
fileSizeLimit: 1024 * 1024,
|
||||
}),
|
||||
downloadCodeOutputBuffer: async () => {
|
||||
throw new Error('Fixture requires prepared bytes');
|
||||
},
|
||||
createDownloadFallback: (input) => ({
|
||||
filename: input.name,
|
||||
filepath: 'fallback',
|
||||
conversationId: input.conversationId,
|
||||
expiresAt: input.expiresAt,
|
||||
}),
|
||||
getRetentionExpiry: async () => ({}),
|
||||
convertImage: async (_req, input, _detail, filename) => {
|
||||
const buffer = await sharp(input).webp({ lossless: true }).toBuffer();
|
||||
return {
|
||||
filepath: await saveBuffer({
|
||||
userId: user,
|
||||
tenantId: req.user.tenantId,
|
||||
buffer,
|
||||
fileName: filename,
|
||||
basePath: 'uploads',
|
||||
}),
|
||||
bytes: buffer.length,
|
||||
};
|
||||
},
|
||||
getStrategyFunctions: () => ({ saveBuffer }),
|
||||
determineFileType: async () => undefined,
|
||||
finalizePreview: async () => null,
|
||||
hasOfficeHtmlPath,
|
||||
sanitizeArtifactPath,
|
||||
flattenArtifactPath,
|
||||
classifyCodeArtifact,
|
||||
extractCodeArtifactText,
|
||||
getExtractedTextFormat,
|
||||
getStorageMetadata,
|
||||
logAxiosError: ({ error }) => {
|
||||
throw error;
|
||||
},
|
||||
logger: { warn: () => undefined, debug: () => undefined },
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
mongoose.deleteModel('File');
|
||||
await mongo?.stop();
|
||||
});
|
||||
|
||||
async function emit(buffer: Buffer, messageId: string, name = 'report.txt', stamp?: number) {
|
||||
const result = await processCodeOutput(
|
||||
{
|
||||
req,
|
||||
id: 'same-live-object',
|
||||
session_id: 'same-live-session',
|
||||
name,
|
||||
messageId,
|
||||
toolCallId: `tool-${messageId}`,
|
||||
conversationId: 'conversation-a',
|
||||
preparedBuffer: buffer,
|
||||
freshClaimAfter: stamp,
|
||||
},
|
||||
deps,
|
||||
);
|
||||
const file = result?.file;
|
||||
if (!file || !('file_id' in file)) throw new Error('Expected a stored snapshot');
|
||||
return file;
|
||||
}
|
||||
|
||||
async function reload(file_id: string) {
|
||||
const file = await mongoose.models.File.findOne({ file_id }).lean<CodeFileCommitData>();
|
||||
if (!file) throw new Error('Snapshot disappeared on reload');
|
||||
const bytes = await readFile(
|
||||
file.storageKey
|
||||
? path.join(directory, 'objects', file.storageKey)
|
||||
: path.join(directory, fromLocalUrlPath(stripCacheBust(file.filepath))),
|
||||
);
|
||||
return { file, bytes };
|
||||
}
|
||||
|
||||
function primingFile(file: IMongoFile): TFile {
|
||||
return {
|
||||
file_id: file.file_id,
|
||||
user: file.user.toString(),
|
||||
tenantId: file.tenantId,
|
||||
filename: file.filename,
|
||||
filepath: file.filepath,
|
||||
type: file.type,
|
||||
bytes: file.bytes,
|
||||
object: 'file',
|
||||
embedded: file.embedded === true,
|
||||
usage: file.usage ?? 1,
|
||||
context: file.context as FileContext,
|
||||
metadata: file.metadata,
|
||||
createdAt: file.createdAt,
|
||||
source: file.source as FileSources,
|
||||
};
|
||||
}
|
||||
|
||||
async function initializeCode(
|
||||
ids: string[],
|
||||
privateIds: string[] = [],
|
||||
deferred = true,
|
||||
failOnceAt?: string,
|
||||
stale = false,
|
||||
) {
|
||||
const files = (
|
||||
(await methods.getFiles({ file_id: { $in: ids }, user, tenantId: req.user.tenantId })) ?? []
|
||||
).map(primingFile);
|
||||
const getFiles = jest.fn<ReturnType<TGetFiles>, Parameters<TGetFiles>>(async (filter) =>
|
||||
((await methods.getFiles({ ...filter, user, tenantId: req.user.tenantId })) ?? []).map(
|
||||
primingFile,
|
||||
),
|
||||
);
|
||||
const liveDirectory = path.join(directory, 'sandbox');
|
||||
const upload = jest.fn(async ({ stream, filename, onUploadStart }: Record<string, unknown>) => {
|
||||
if (!(stream instanceof Readable) || typeof filename !== 'string')
|
||||
throw new Error('Invalid upload');
|
||||
if (filename === failOnceAt) {
|
||||
failOnceAt = undefined;
|
||||
throw Object.assign(new Error('Upload rejected'), { response: { status: 400 } });
|
||||
}
|
||||
if (typeof onUploadStart === 'function') onUploadStart();
|
||||
await writeLocalFile(
|
||||
path.join(liveDirectory, path.dirname(filename)),
|
||||
path.basename(filename),
|
||||
await readStream(stream),
|
||||
);
|
||||
return {
|
||||
storage_session_id: 'same-live-session',
|
||||
file_id: filename === 'report.txt' ? 'same-live-object' : filename,
|
||||
filename,
|
||||
};
|
||||
});
|
||||
const service = createProvisionService({
|
||||
getStrategyFunctions: (source) =>
|
||||
source === FileSources.execute_code
|
||||
? { handleFileUpload: upload }
|
||||
: {
|
||||
getDownloadStream: async (_req, filepath) =>
|
||||
createReadStream(path.join(directory, fromLocalUrlPath(stripCacheBust(filepath)))),
|
||||
},
|
||||
uploadVectors: async () => ({ embedded: true }),
|
||||
loadAuthValues: async () => ({}),
|
||||
});
|
||||
const checkSessionsAlive = jest.fn(service.checkSessionsAlive);
|
||||
const primed = await primeResources({
|
||||
req,
|
||||
getFiles,
|
||||
requestFileSet: new Set(),
|
||||
attachments: deferred ? undefined : Promise.resolve(files),
|
||||
provisionCandidates: deferred ? files : undefined,
|
||||
tool_resources: privateIds.length ? { execute_code: { file_ids: privateIds } } : {},
|
||||
agentId: 'agent-a',
|
||||
enabledToolResources: new Set([EToolResources.execute_code]),
|
||||
checkSessionsAlive,
|
||||
});
|
||||
const context: CodeFileAgent = {
|
||||
id: 'agent-a',
|
||||
tool_resources: primed.tool_resources,
|
||||
provisionState: primed.provisionState,
|
||||
fileConsumers: { executeCode: true, fileSearch: false },
|
||||
};
|
||||
// Eager tool loading runs before the lazy callback, as initializeAgent does.
|
||||
const eager = await selectCodeFiles({
|
||||
files: context.tool_resources?.execute_code?.files ?? [],
|
||||
privateFileIds: new Set(privateIds),
|
||||
routeKey: 'default',
|
||||
getFileInfo: async () => (stale ? null : { lastModified: new Date().toISOString() }),
|
||||
});
|
||||
for (const selected of eager.selected) {
|
||||
if (!selected.isActive)
|
||||
await writeLocalFile(
|
||||
path.join(liveDirectory, path.dirname(selected.sandboxName)),
|
||||
path.basename(selected.sandboxName),
|
||||
(await reload(selected.file.file_id)).bytes,
|
||||
);
|
||||
}
|
||||
prepareQueuedCodeFileContext(context, [context], user);
|
||||
const callback = createProvisionFilesCallback({
|
||||
req,
|
||||
fileStorage: deps.fileStorage,
|
||||
agentToolContexts: new Map([['agent-a', context]]),
|
||||
provisionToCodeEnv: service.provisionToCodeEnv,
|
||||
provisionToVectorDB: service.provisionToVectorDB,
|
||||
updateFile: async () => undefined,
|
||||
updateCodeEnvRef: async () => undefined,
|
||||
addEmbeddedEntity: async () => undefined,
|
||||
});
|
||||
const advertised = new Map(context.provisionState?.codeEnvDestinations);
|
||||
let mounted: Awaited<ReturnType<typeof callback>>;
|
||||
if (failOnceAt) {
|
||||
await expect(callback(['execute_code'], 'agent-a')).rejects.toThrow();
|
||||
expect(context.provisionState?.codeEnvDestinations).toEqual(advertised);
|
||||
mounted = await callback(['execute_code'], 'agent-a');
|
||||
expect(new Set(mounted.map((file) => file.name))).toEqual(new Set(advertised.values()));
|
||||
} else mounted = await callback(['execute_code'], 'agent-a');
|
||||
expect(getFiles).toHaveBeenCalledTimes(privateIds.length ? 1 : 0);
|
||||
expect(checkSessionsAlive.mock.calls.length).toBeLessThanOrEqual(1);
|
||||
const destinations = new Map(
|
||||
eager.selected.map((selected) => [selected.file.file_id, selected.sandboxName]),
|
||||
);
|
||||
for (const [id, name] of advertised) destinations.set(id, name);
|
||||
return { context, upload, advertised, mounted, destinations };
|
||||
}
|
||||
|
||||
it.each([true, false])(
|
||||
'initializes the newest snapshot before eager/lazy partitioning (deferred %s)',
|
||||
async (deferred) => {
|
||||
const first = await emit(Buffer.from('Version 1'), 'message-1', 'report.txt', 100);
|
||||
await initializeCode([first.file_id], [], deferred);
|
||||
expect((await reload(first.file_id)).file.metadata?.codeEnvRef).toBeDefined();
|
||||
const second = await emit(Buffer.from('Version 2'), 'message-2', 'report.txt', 200);
|
||||
const next = await initializeCode([first.file_id, second.file_id], [], deferred);
|
||||
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
|
||||
'Version 2',
|
||||
);
|
||||
expect(next.advertised.get(second.file_id)).toBe('report.txt');
|
||||
expect(next.mounted.map((file) => file.name)).toEqual(['report.txt']);
|
||||
expect(next.context.tool_resources?.execute_code?.files?.map((file) => file.file_id)).toEqual(
|
||||
[second.file_id],
|
||||
);
|
||||
await initializeCode([first.file_id], [], deferred);
|
||||
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
|
||||
'Version 1',
|
||||
);
|
||||
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps tenant authorization and private collisions through initialization and lazy setup', async () => {
|
||||
const first = await emit(Buffer.from('Version 1'), 'message-1', 'report.txt', 100);
|
||||
await initializeCode([first.file_id]);
|
||||
const second = await emit(Buffer.from('Version 2'), 'message-2', 'report.txt', 200);
|
||||
req.user.tenantId = 'tenant-b';
|
||||
const foreign = await emit(Buffer.from('Foreign tenant'), 'foreign', 'report.txt', 300);
|
||||
req.user.tenantId = 'tenant-a';
|
||||
const privateFile = await emit(Buffer.from('Private setup'), 'private', 'report.txt', 400);
|
||||
await methods.updateFile({ file_id: privateFile.file_id, context: FileContext.agents });
|
||||
const next = await initializeCode(
|
||||
[first.file_id, second.file_id, foreign.file_id],
|
||||
[privateFile.file_id],
|
||||
);
|
||||
const name = next.advertised.get(second.file_id)!;
|
||||
expect(name).toBe('report.txt');
|
||||
expect((await readFile(path.join(directory, 'sandbox', name))).toString()).toBe('Version 2');
|
||||
const privateName = next.advertised.get(privateFile.file_id)!;
|
||||
expect(privateName).not.toBe(name);
|
||||
expect((await readFile(path.join(directory, 'sandbox', privateName))).toString()).toBe(
|
||||
'Private setup',
|
||||
);
|
||||
expect(next.advertised.has(foreign.file_id)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[true, true],
|
||||
[true, false],
|
||||
[false, true],
|
||||
[false, false],
|
||||
] as const)(
|
||||
'keeps shared snapshots ahead of an already primed private resource (deferred %s, named %s)',
|
||||
async (deferred, named) => {
|
||||
const privateFile = await emit(Buffer.from('Private setup'), 'private', 'report.txt', 50);
|
||||
await methods.updateFile({ file_id: privateFile.file_id, context: FileContext.agents });
|
||||
await initializeCode([], [privateFile.file_id], deferred);
|
||||
expect((await reload(privateFile.file_id)).file.metadata?.codeEnvRef).toMatchObject({
|
||||
kind: 'agent',
|
||||
id: 'agent-a',
|
||||
sandboxFilename: 'report.txt',
|
||||
});
|
||||
if (!named)
|
||||
await mongoose.models.File.updateOne(
|
||||
{ file_id: privateFile.file_id },
|
||||
{
|
||||
$unset: {
|
||||
'metadata.codeEnvRef.sandboxFilename': '',
|
||||
'metadata.codeEnvRefs.default.sandboxFilename': '',
|
||||
},
|
||||
},
|
||||
);
|
||||
const first = await emit(Buffer.from('Shared V1'), 'shared-1', 'report.txt', 100);
|
||||
const alone = await initializeCode([first.file_id], [], deferred);
|
||||
const next = await initializeCode([first.file_id], [privateFile.file_id], deferred);
|
||||
expect(next.destinations.get(first.file_id)).toBe(alone.destinations.get(first.file_id));
|
||||
expect(next.destinations.get(first.file_id)).toBe('report.txt');
|
||||
const privateName = next.destinations.get(privateFile.file_id)!;
|
||||
expect(privateName).toMatch(/^report-.+\.txt$/);
|
||||
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
|
||||
'Shared V1',
|
||||
);
|
||||
expect((await readFile(path.join(directory, 'sandbox', privateName))).toString()).toBe(
|
||||
'Private setup',
|
||||
);
|
||||
expect((await reload(privateFile.file_id)).file.metadata?.codeEnvRef?.sandboxFilename).toBe(
|
||||
privateName,
|
||||
);
|
||||
const second = await emit(Buffer.from('Shared V2'), 'shared-2', 'report.txt', 200);
|
||||
for (const ids of [
|
||||
[first.file_id, second.file_id],
|
||||
[first.file_id],
|
||||
[first.file_id, second.file_id],
|
||||
]) {
|
||||
const restored = await initializeCode(ids, [privateFile.file_id], deferred);
|
||||
expect(restored.destinations.get(privateFile.file_id)).toBe(privateName);
|
||||
expect(restored.destinations.get(ids[ids.length - 1])).toBe('report.txt');
|
||||
expect((await readFile(path.join(directory, 'sandbox/report.txt'))).toString()).toBe(
|
||||
ids.length === 1 ? 'Shared V1' : 'Shared V2',
|
||||
);
|
||||
expect((await readFile(path.join(directory, 'sandbox', privateName))).toString()).toBe(
|
||||
'Private setup',
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['report.txt', 'nested/report-alias.txt'])(
|
||||
'restores a rewritten upload at its original destination %s on old branches',
|
||||
async (name) => {
|
||||
const uploaded = await emit(Buffer.from('Uploaded original'), 'upload', 'upload.txt', 50);
|
||||
await methods.updateFile({
|
||||
file_id: uploaded.file_id,
|
||||
context: FileContext.message_attachment,
|
||||
metadata: {
|
||||
codeDestination: true,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: user,
|
||||
storage_session_id: 'same-live-session',
|
||||
file_id: 'same-live-object',
|
||||
sandboxFilename: name,
|
||||
},
|
||||
},
|
||||
});
|
||||
await mongoose.models.File.updateOne(
|
||||
{ file_id: uploaded.file_id },
|
||||
{
|
||||
$unset: { 'metadata.codeEnvSource': '', 'metadata.sourceDispatchedAt': '' },
|
||||
$set: { createdAt: new Date(50) },
|
||||
},
|
||||
{ timestamps: false },
|
||||
);
|
||||
const live = path.join(directory, 'sandbox', name);
|
||||
await writeLocalFile(path.dirname(live), path.basename(live), Buffer.from('Version 1'));
|
||||
const first = await emit(await readFile(live), 'message-1', name, 100);
|
||||
await initializeCode([uploaded.file_id, first.file_id]);
|
||||
await writeLocalFile(path.dirname(live), path.basename(live), Buffer.from('Version 2'));
|
||||
const second = await emit(await readFile(live), 'message-2', name, 200);
|
||||
await initializeCode([uploaded.file_id, first.file_id, second.file_id]);
|
||||
expect((await readFile(path.join(directory, 'sandbox', name))).toString()).toBe('Version 2');
|
||||
const restored = await initializeCode([uploaded.file_id, first.file_id]);
|
||||
expect((await readFile(path.join(directory, 'sandbox', name))).toString()).toBe('Version 1');
|
||||
expect(
|
||||
restored.context.tool_resources?.execute_code?.files?.map((file) => file.file_id),
|
||||
).toEqual([first.file_id]);
|
||||
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
'restores a literal WebP rewrite ahead of a stale converted legacy receipt (deferred %s)',
|
||||
async (deferred) => {
|
||||
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
const green = await sharp({
|
||||
create: { width: 2, height: 2, channels: 3, background: 'green' },
|
||||
})
|
||||
.webp()
|
||||
.toBuffer();
|
||||
const legacy = await emit(blue, 'legacy', 'nested/chart.png', 100);
|
||||
await methods.updateFile({ file_id: legacy.file_id, context: FileContext.execute_code });
|
||||
await mongoose.models.File.updateOne(
|
||||
{ file_id: legacy.file_id },
|
||||
{ $unset: { 'metadata.codeEnvRef': '', 'metadata.codeEnvRefs': '' } },
|
||||
);
|
||||
const original = await initializeCode([legacy.file_id], [], deferred, undefined, true);
|
||||
const name = original.destinations.get(legacy.file_id)!;
|
||||
expect(name).toBe('nested/chart.webp');
|
||||
expect((await reload(legacy.file_id)).file.metadata?.codeEnvRef?.sandboxFilename).toBe(name);
|
||||
const live = path.join(directory, 'sandbox', name);
|
||||
await writeLocalFile(path.dirname(live), path.basename(live), green);
|
||||
const edited = await emit(await readFile(live), 'edited', name, 200);
|
||||
const oldBytes = (await reload(legacy.file_id)).bytes;
|
||||
const newBytes = (await reload(edited.file_id)).bytes;
|
||||
expect(oldBytes).not.toEqual(newBytes);
|
||||
let alias: string | undefined;
|
||||
for (const ids of [
|
||||
[legacy.file_id, edited.file_id],
|
||||
[legacy.file_id, edited.file_id],
|
||||
[legacy.file_id],
|
||||
[legacy.file_id, edited.file_id],
|
||||
]) {
|
||||
await rm(path.join(directory, 'sandbox'), { recursive: true, force: true });
|
||||
const restored = await initializeCode(ids, [], deferred, undefined, true);
|
||||
expect(restored.destinations.size).toBe(ids.length);
|
||||
if (ids.length === 1) {
|
||||
expect(restored.destinations.get(legacy.file_id)).toBe(name);
|
||||
expect(await readFile(live)).toEqual(oldBytes);
|
||||
} else {
|
||||
expect(restored.destinations.get(edited.file_id)).toBe(name);
|
||||
const legacyName = restored.destinations.get(legacy.file_id)!;
|
||||
expect(legacyName).toMatch(/^nested\/chart-.+\.webp$/);
|
||||
if (alias) expect(legacyName).toBe(alias);
|
||||
alias = legacyName;
|
||||
expect(await readFile(live)).toEqual(newBytes);
|
||||
expect(await readFile(path.join(directory, 'sandbox', legacyName))).toEqual(oldBytes);
|
||||
}
|
||||
expect((await reload(legacy.file_id)).bytes).toEqual(oldBytes);
|
||||
expect((await reload(edited.file_id)).bytes).toEqual(newBytes);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('retains distinct captured image paths across conversion, lazy mounts, eager reload and branch return', async () => {
|
||||
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
const red = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'red' } })
|
||||
.jpeg()
|
||||
.toBuffer();
|
||||
const first = await emit(blue, 'png', 'nested/chart.png', 100);
|
||||
await initializeCode([first.file_id]);
|
||||
const second = await emit(red, 'jpeg', 'nested/chart.jpeg', 200);
|
||||
const next = await initializeCode([first.file_id, second.file_id]);
|
||||
expect(next.destinations.size).toBe(2);
|
||||
expect(new Set(next.destinations.values()).size).toBe(2);
|
||||
const pngName = next.destinations.get(first.file_id)!;
|
||||
const jpegName = next.destinations.get(second.file_id)!;
|
||||
expect(pngName).toBe('nested/chart.webp');
|
||||
expect(jpegName).toMatch(/^nested\/chart-.+\.webp$/);
|
||||
const pngBytes = (await reload(first.file_id)).bytes;
|
||||
const jpegBytes = (await reload(second.file_id)).bytes;
|
||||
expect(await readFile(path.join(directory, 'sandbox', pngName))).toEqual(pngBytes);
|
||||
expect(await readFile(path.join(directory, 'sandbox', jpegName))).toEqual(jpegBytes);
|
||||
const eager = await initializeCode([first.file_id, second.file_id]);
|
||||
expect(eager.destinations).toEqual(next.destinations);
|
||||
const rewritten = await emit(red, 'png-rewritten', 'nested/chart.png', 300);
|
||||
const newer = await initializeCode([first.file_id, second.file_id, rewritten.file_id]);
|
||||
expect(newer.destinations.has(first.file_id)).toBe(false);
|
||||
expect(newer.destinations.size).toBe(2);
|
||||
await initializeCode([first.file_id, second.file_id]);
|
||||
expect(await readFile(path.join(directory, 'sandbox', pngName))).toEqual(pngBytes);
|
||||
expect(await readFile(path.join(directory, 'sandbox', jpegName))).toEqual(jpegBytes);
|
||||
});
|
||||
|
||||
it.each([0, 1])(
|
||||
'gives a new literal write ownership of assigned image mount %s without merging captures',
|
||||
async (target) => {
|
||||
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
const red = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'red' } })
|
||||
.jpeg()
|
||||
.toBuffer();
|
||||
const green = await sharp({
|
||||
create: { width: 2, height: 2, channels: 3, background: 'green' },
|
||||
})
|
||||
.webp()
|
||||
.toBuffer();
|
||||
const a = await emit(blue, 'png', 'nested/chart.png', 100);
|
||||
await initializeCode([a.file_id]);
|
||||
const b = await emit(red, 'jpeg', 'nested/chart.jpeg', 200);
|
||||
const old = await initializeCode([a.file_id, b.file_id]);
|
||||
const original = target === 0 ? a : b;
|
||||
const editedPath = old.destinations.get(original.file_id)!;
|
||||
const live = path.join(directory, 'sandbox', editedPath);
|
||||
await writeLocalFile(path.dirname(live), path.basename(live), green);
|
||||
const edited = await emit(await readFile(live), 'edited', editedPath, 300);
|
||||
const next = await initializeCode(
|
||||
[a.file_id, b.file_id, edited.file_id],
|
||||
[],
|
||||
true,
|
||||
target === 1 ? editedPath : undefined,
|
||||
);
|
||||
expect(next.destinations.size).toBe(3);
|
||||
expect(next.destinations.get(edited.file_id)).toBe(editedPath);
|
||||
expect(next.destinations.get(original.file_id)).not.toBe(editedPath);
|
||||
expect(await readFile(live)).toEqual((await reload(edited.file_id)).bytes);
|
||||
expect(
|
||||
await readFile(path.join(directory, 'sandbox', next.destinations.get(original.file_id)!)),
|
||||
).toEqual((await reload(original.file_id)).bytes);
|
||||
const again = await initializeCode([a.file_id, b.file_id, edited.file_id]);
|
||||
expect(again.destinations).toEqual(next.destinations);
|
||||
const branch = await initializeCode([a.file_id, b.file_id]);
|
||||
expect(branch.destinations.size).toBe(2);
|
||||
for (const file of [a, b])
|
||||
expect(
|
||||
await readFile(path.join(directory, 'sandbox', branch.destinations.get(file.file_id)!)),
|
||||
).toEqual((await reload(file.file_id)).bytes);
|
||||
const returned = await initializeCode([a.file_id, b.file_id, edited.file_id]);
|
||||
expect(returned.destinations).toEqual(next.destinations);
|
||||
expect(await readFile(live)).toEqual((await reload(edited.file_id)).bytes);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['report.txt', 'published/kept-name.txt'])(
|
||||
'mounts the owned published V1 copy instead of mutable provenance at %s',
|
||||
async (name) => {
|
||||
const scope = {
|
||||
userId: user,
|
||||
tenantId: 'tenant-a',
|
||||
conversationId: 'conversation-a',
|
||||
runId: 'run-publish',
|
||||
executionId: 'child',
|
||||
agentId: 'agent-a',
|
||||
sourceFileId: 'mutable-source',
|
||||
};
|
||||
const result = await processCodeOutput(
|
||||
{
|
||||
req,
|
||||
id: scope.sourceFileId,
|
||||
session_id: 'source-session',
|
||||
name: 'report.txt',
|
||||
conversationId: scope.conversationId,
|
||||
messageId: 'published-message',
|
||||
preparedBuffer: Buffer.from('Published V1'),
|
||||
publication: {
|
||||
scope,
|
||||
provenance: {
|
||||
runId: scope.runId,
|
||||
executionId: scope.executionId,
|
||||
agentId: scope.agentId,
|
||||
sourceFileId: scope.sourceFileId,
|
||||
publishedAt: new Date().toISOString(),
|
||||
inputFileIds: [],
|
||||
},
|
||||
publish: methods.publishRunArtifactFile,
|
||||
find: methods.findRunArtifactFile,
|
||||
discard: async () => undefined,
|
||||
},
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if (!result?.file || !('file_id' in result.file)) throw new Error('Expected publication');
|
||||
const published = result.file;
|
||||
const unprovisioned = (await methods.getFiles({ file_id: published.file_id }))![0];
|
||||
const probe = jest.fn(async () => ({ lastModified: new Date().toISOString() }));
|
||||
const owned = {
|
||||
kind: 'user' as const,
|
||||
id: user,
|
||||
storage_session_id: 'owned-session',
|
||||
file_id: 'owned-copy',
|
||||
sandboxFilename: name,
|
||||
executionProfile: 'default' as const,
|
||||
};
|
||||
await withFileProvisioning({
|
||||
file: published,
|
||||
userId: user,
|
||||
storage: deps.fileStorage,
|
||||
destination: 'published-copy',
|
||||
operation: async (_claim, onUploadStart) => {
|
||||
onUploadStart();
|
||||
await writeLocalFile(directory, 'owned-copy', (await reload(published.file_id)).bytes);
|
||||
return {
|
||||
value: owned,
|
||||
commit: { code: { routeKey: 'default', ref: owned, legacyRef: owned } },
|
||||
};
|
||||
},
|
||||
});
|
||||
await writeLocalFile(directory, 'mutable-source', Buffer.from('Mutable V2'));
|
||||
const record = (await methods.getFiles({ file_id: published.file_id }))![0];
|
||||
const getFileInfo = jest.fn(async (ref: { file_id: string }) => ({
|
||||
lastModified: new Date().toISOString(),
|
||||
originalFilename: ref.file_id === 'owned-copy' ? name : 'wrong-live-name.txt',
|
||||
}));
|
||||
const selected = await selectCodeFiles({
|
||||
files: [primingFile(record)],
|
||||
routeKey: 'default',
|
||||
getFileInfo,
|
||||
});
|
||||
expect(selected.selected).toHaveLength(1);
|
||||
const mount = selected.selected[0];
|
||||
expect(mount.isActive).toBe(true);
|
||||
expect(await readFile(path.join(directory, mount.sourceRef.file_id!))).toEqual(
|
||||
Buffer.from('Published V1'),
|
||||
);
|
||||
expect(mount.sourceRef).toMatchObject(owned);
|
||||
expect(mount.sandboxName).toBe(name);
|
||||
expect(getFileInfo).toHaveBeenCalledTimes(1);
|
||||
expect(getFileInfo).toHaveBeenCalledWith(owned);
|
||||
expect(record.context).toBe(FileContext.run_artifact);
|
||||
expect(record.metadata?.codeEnvSource?.file_id).toBe('mutable-source');
|
||||
expect(
|
||||
(
|
||||
await selectCodeFiles({
|
||||
files: [primingFile(unprovisioned)],
|
||||
routeKey: 'default',
|
||||
getFileInfo: probe,
|
||||
})
|
||||
).selected,
|
||||
).toEqual([]);
|
||||
expect(probe).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['no-storage-writer', 'processing-before-upload'] as const)(
|
||||
'does not retain an unfinished snapshot claim after %s',
|
||||
async (mode) => {
|
||||
const kept = await emit(Buffer.from('Committed V1'), 'kept');
|
||||
if (mode === 'no-storage-writer') deps.getStrategyFunctions = () => ({});
|
||||
else
|
||||
deps.determineFileType = async () => {
|
||||
throw new Error('Processing failed before upload');
|
||||
};
|
||||
const attempt = processCodeOutput(
|
||||
{
|
||||
req,
|
||||
id: 'fallback-source',
|
||||
session_id: 'fallback-session',
|
||||
name: 'report.txt',
|
||||
messageId: 'fallback',
|
||||
conversationId: 'conversation-a',
|
||||
preparedBuffer: Buffer.from('Fallback bytes'),
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if (mode === 'processing-before-upload')
|
||||
await expect(attempt).rejects.toThrow('Processing failed before upload');
|
||||
else expect((await attempt)?.file?.filepath).toBe('fallback');
|
||||
expect((await reload(kept.file_id)).bytes.toString()).toBe('Committed V1');
|
||||
expect(
|
||||
await mongoose.models.File.countDocuments({
|
||||
context: FileContext.execute_code_snapshot,
|
||||
user,
|
||||
tenantId: 'tenant-a',
|
||||
}),
|
||||
).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
it('retains the unfinished claim after an uncertain external write', async () => {
|
||||
const kept = await emit(Buffer.from('Committed V1'), 'kept');
|
||||
const { saveBuffer } = deps.getStrategyFunctions(FileSources.local);
|
||||
deps.getStrategyFunctions = () => ({
|
||||
saveBuffer: async (input) => {
|
||||
await saveBuffer!(input);
|
||||
throw new Error('Storage acknowledgement lost');
|
||||
},
|
||||
});
|
||||
await expect(emit(Buffer.from('Uncertain V2'), 'uncertain')).rejects.toThrow(
|
||||
'File provisioning outcome is unknown',
|
||||
);
|
||||
const records = await mongoose.models.File.find({
|
||||
context: FileContext.execute_code_snapshot,
|
||||
user,
|
||||
tenantId: 'tenant-a',
|
||||
}).lean<IMongoFile[]>();
|
||||
expect(records).toHaveLength(2);
|
||||
const unfinished = records.find((file) => file.file_id !== kept.file_id)!;
|
||||
expect(unfinished.filepath).toBeUndefined();
|
||||
expect(unfinished.storageOperations?.provisioning).toHaveLength(1);
|
||||
await expect(
|
||||
deps.fileStorage.recovery.recover(
|
||||
{ file_id: unfinished.file_id, userId: user, tenantId: 'tenant-a' },
|
||||
deps.fileStorage,
|
||||
),
|
||||
).rejects.toThrow('File provisioning outcome is unknown');
|
||||
expect(await mongoose.models.File.findOne({ file_id: unfinished.file_id }).lean()).toEqual(
|
||||
unfinished,
|
||||
);
|
||||
expect((await reload(kept.file_id)).bytes.toString()).toBe('Committed V1');
|
||||
});
|
||||
|
||||
it('retains V1 and V2 bytes and message references after same-path rewrites and reload', async () => {
|
||||
const first = await emit(Buffer.from('Version 1'), 'message-1');
|
||||
const second = await emit(Buffer.from('Version 2'), 'message-2');
|
||||
const references = JSON.parse(JSON.stringify([first, second]));
|
||||
expect((await reload(references[0].file_id)).bytes.toString()).toBe('Version 1');
|
||||
expect((await reload(references[1].file_id)).bytes.toString()).toBe('Version 2');
|
||||
expect(first.file_id).not.toBe(second.file_id);
|
||||
expect((await reload(first.file_id)).file.messageId).toBe('message-1');
|
||||
expect((await reload(second.file_id)).file.messageId).toBe('message-2');
|
||||
expect(first.filename).toBe('report.txt');
|
||||
expect(second.filename).toBe('report.txt');
|
||||
});
|
||||
|
||||
it('keeps each image generation after conversion and reload', async () => {
|
||||
const blue = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'blue' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
const red = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'red' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
const first = await emit(blue, 'image-1', 'nested/chart.png');
|
||||
const second = await emit(red, 'image-2', 'nested/chart.png');
|
||||
expect((await reload(first.file_id)).bytes).toEqual(
|
||||
await sharp(blue).webp({ lossless: true }).toBuffer(),
|
||||
);
|
||||
expect((await reload(second.file_id)).bytes).toEqual(
|
||||
await sharp(red).webp({ lossless: true }).toBuffer(),
|
||||
);
|
||||
expect(first.filepath).not.toBe(second.filepath);
|
||||
});
|
||||
|
||||
it('isolates simultaneous same-name outputs, including identical reads and older dispatches', async () => {
|
||||
const values = ['Version 1', 'Version 2', 'Version 2', 'Version 0'];
|
||||
const files = await Promise.all(
|
||||
values.map((value, index) =>
|
||||
emit(Buffer.from(value), `message-${index}`, 'report.txt', 400 - index),
|
||||
),
|
||||
);
|
||||
expect(new Set(files.map((file) => file.file_id)).size).toBe(values.length);
|
||||
for (const [index, file] of files.entries()) {
|
||||
expect((await reload(file.file_id)).bytes.toString()).toBe(values[index]);
|
||||
expect((await reload(file.file_id)).file.user.toString()).toBe(user);
|
||||
expect((await reload(file.file_id)).file.tenantId).toBe('tenant-a');
|
||||
}
|
||||
});
|
||||
|
||||
it.each([FileSources.s3, FileSources.cloudfront] as const)(
|
||||
'keeps immutable tenant-owned object keys with %s storage',
|
||||
async (source) => {
|
||||
const client = new S3Client({ region: 'us-east-1' });
|
||||
const remote = mockClient(client);
|
||||
jest.spyOn(s3, 'initializeS3').mockReturnValue(client);
|
||||
remote.on(PutObjectCommand).callsFake(async ({ Key, Body }) => {
|
||||
const location = path.join(directory, 'objects', Key);
|
||||
await writeLocalFile(path.dirname(location), path.basename(location), Body);
|
||||
return {};
|
||||
});
|
||||
req.config.fileStrategy = source;
|
||||
deps.getStrategyFunctions = () => ({
|
||||
saveBuffer: (input) =>
|
||||
saveBufferToS3({
|
||||
...input,
|
||||
urlBuilder: async (options) =>
|
||||
`https://objects.example/${getS3Key({ ...options, basePath: options.basePath ?? 'images' })}`,
|
||||
}),
|
||||
});
|
||||
try {
|
||||
const first = await emit(Buffer.from('Version 1'), 'cloud-1');
|
||||
const second = await emit(Buffer.from('Version 2'), 'cloud-2');
|
||||
const original = await reload(first.file_id);
|
||||
const rewritten = await reload(second.file_id);
|
||||
expect(original.bytes.toString()).toBe('Version 1');
|
||||
expect(rewritten.bytes.toString()).toBe('Version 2');
|
||||
expect(original.file.storageKey).not.toBe(rewritten.file.storageKey);
|
||||
expect(original.file.storageKey).toContain('tenant-a');
|
||||
expect(original.file.storageKey).toContain(user);
|
||||
expect(remote.commandCalls(PutObjectCommand)).toHaveLength(2);
|
||||
} finally {
|
||||
remote.restore();
|
||||
client.destroy();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('hydrates only message-referenced snapshots in the authenticated tenant and keeps the newest live path', async () => {
|
||||
const first = await emit(Buffer.from('Version 1'), 'message-1', 'report.txt', 100);
|
||||
const second = await emit(Buffer.from('Version 2'), 'message-2', 'report.txt', 200);
|
||||
const late = await emit(Buffer.from('Version 0'), 'late-harvest', 'report.txt', 50);
|
||||
const owner = { userId: user, tenantId: 'tenant-a' };
|
||||
const files = await methods.getCodeGeneratedFiles(
|
||||
'conversation-a',
|
||||
[first.file_id, second.file_id, late.file_id],
|
||||
owner,
|
||||
);
|
||||
expect(files).toHaveLength(3);
|
||||
expect(
|
||||
await methods.getCodeGeneratedFiles('conversation-a', [first.file_id], {
|
||||
...owner,
|
||||
tenantId: 'other-tenant',
|
||||
}),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await methods.getCodeGeneratedFiles('conversation-a', [first.file_id], {
|
||||
...owner,
|
||||
userId: new mongoose.Types.ObjectId().toString(),
|
||||
}),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
(await methods.getCodeGeneratedFiles('conversation-a', [first.file_id], owner)).map(
|
||||
(file) => file.file_id,
|
||||
),
|
||||
).toEqual([first.file_id]);
|
||||
const getFileInfo = jest.fn(async () => ({
|
||||
lastModified: new Date().toISOString(),
|
||||
originalFilename: 'report.txt',
|
||||
}));
|
||||
const selected = await selectCodeFiles({
|
||||
files: files.map((file) => ({
|
||||
user: file.user.toString(),
|
||||
file_id: file.file_id,
|
||||
filename: file.filename,
|
||||
filepath: file.filepath,
|
||||
bytes: file.bytes,
|
||||
type: file.type,
|
||||
object: 'file',
|
||||
embedded: false,
|
||||
usage: file.usage,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
createdAt: file.createdAt,
|
||||
metadata: file.metadata,
|
||||
})),
|
||||
routeKey: 'default',
|
||||
getFileInfo,
|
||||
});
|
||||
expect(selected.selected.map(({ file }) => file.file_id)).toEqual([second.file_id]);
|
||||
expect(selected.selected[0].sandboxName).toBe('report.txt');
|
||||
expect(selected.selected[0].isActive).toBe(false);
|
||||
expect(getFileInfo).not.toHaveBeenCalled();
|
||||
expect((await reload(first.file_id)).bytes.toString()).toBe('Version 1');
|
||||
});
|
||||
|
||||
it('never reuses or deletes mutable recovery receipts after prime and a later rewrite', async () => {
|
||||
const liveDirectory = path.join(directory, 'sandbox');
|
||||
const livePath = path.join(liveDirectory, 'report.txt');
|
||||
await writeLocalFile(liveDirectory, 'report.txt', Buffer.from('Version 1'));
|
||||
const first = await emit(await readFile(livePath), 'message-1');
|
||||
const ref = first.metadata!.codeEnvSource!;
|
||||
await withFileProvisioning({
|
||||
file: first,
|
||||
userId: user,
|
||||
storage: deps.fileStorage,
|
||||
destination: 'code:default:report.txt',
|
||||
operation: async (_claim, onUploadStart) => {
|
||||
onUploadStart();
|
||||
await writeLocalFile(liveDirectory, 'report.txt', (await reload(first.file_id)).bytes);
|
||||
return { value: ref, commit: { code: { routeKey: 'default', ref, legacyRef: ref } } };
|
||||
},
|
||||
});
|
||||
await writeLocalFile(liveDirectory, 'report.txt', Buffer.from('Version 2'));
|
||||
const second = await emit(await readFile(livePath), 'message-2');
|
||||
const original = (await reload(first.file_id)).file;
|
||||
const getFileInfo = jest.fn(async () => ({ lastModified: new Date().toISOString() }));
|
||||
const selected = await selectCodeFiles({
|
||||
files: [
|
||||
{
|
||||
file_id: original.file_id,
|
||||
filename: original.filename,
|
||||
filepath: original.filepath,
|
||||
bytes: original.bytes,
|
||||
type: original.type,
|
||||
metadata: original.metadata,
|
||||
user: original.user.toString(),
|
||||
object: 'file',
|
||||
embedded: false,
|
||||
usage: original.usage ?? 1,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
},
|
||||
],
|
||||
routeKey: 'default',
|
||||
getFileInfo,
|
||||
});
|
||||
expect(selected.selected).toHaveLength(1);
|
||||
expect(selected.selected[0].isActive).toBe(false);
|
||||
expect(selected.selected[0].ref).toBeUndefined();
|
||||
expect(selected.selected[0].sandboxName).toBe('report.txt');
|
||||
expect(getFileInfo).not.toHaveBeenCalled();
|
||||
expect((await reload(first.file_id)).bytes.toString()).toBe('Version 1');
|
||||
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
|
||||
const httpClient = jest.fn(async () => undefined);
|
||||
const remove = createCodeFileDeleter({
|
||||
httpClient,
|
||||
getAuthHeaders: async () => ({}),
|
||||
getBaseUrl: () => 'https://code.example',
|
||||
routeKey: () => 'default',
|
||||
profileHeaders: () => ({}),
|
||||
downloadQuery: () => '',
|
||||
clearFileCodeEnvRefs: methods.clearFileCodeEnvRefs,
|
||||
logger: { warn: () => undefined },
|
||||
});
|
||||
expect(original.metadata?.codeEnvRef).toMatchObject({ file_id: 'same-live-object' });
|
||||
const removePrimary = createLocalFileDeleter({
|
||||
deleteRagFile: async () => undefined,
|
||||
logger: { warn: () => undefined, error: () => undefined },
|
||||
});
|
||||
const cleanupReq = {
|
||||
user: req.user,
|
||||
config: { fileConfig: { retainedFileCleanup: true } },
|
||||
};
|
||||
const removeSnapshot = createDeleteFileWithSecondaryStorage<
|
||||
typeof cleanupReq,
|
||||
typeof original,
|
||||
never
|
||||
>({
|
||||
source: FileSources.local,
|
||||
deleteFile: () => removePrimary({ user: req.user, config: { paths: paths() } }, original),
|
||||
getDeleteFile: () => (_request, file) => remove(req, file),
|
||||
clearFileEmbeddings: methods.clearFileEmbeddings,
|
||||
isFileStorageClean: methods.isFileStorageClean,
|
||||
fileStorage: { ...methods, recovery: deps.fileStorage.recovery },
|
||||
logger: { warn: () => undefined },
|
||||
});
|
||||
await removeSnapshot(cleanupReq, original);
|
||||
expect(httpClient).not.toHaveBeenCalled();
|
||||
expect((await readFile(livePath)).toString()).toBe('Version 2');
|
||||
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
|
||||
const reconciled = await mongoose.models.File.findOne({
|
||||
file_id: original.file_id,
|
||||
}).lean<CodeFileCommitData>();
|
||||
expect(reconciled?.metadata?.codeEnvRef).toBeUndefined();
|
||||
expect(reconciled?.metadata?.codeEnvRefs?.default).toBeUndefined();
|
||||
});
|
||||
|
||||
it('deletes one primary snapshot without deleting another version', async () => {
|
||||
const first = await emit(Buffer.from('Version 1'), 'message-1');
|
||||
const second = await emit(Buffer.from('Version 2'), 'message-2');
|
||||
const remove = createLocalFileDeleter({
|
||||
deleteRagFile: async () => undefined,
|
||||
logger: { warn: () => undefined, error: () => undefined },
|
||||
});
|
||||
await remove({ user: req.user, config: { paths: paths() } }, first);
|
||||
expect((await reload(second.file_id)).bytes.toString()).toBe('Version 2');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,12 +1,6 @@
|
|||
import path from 'path';
|
||||
import { v4 } from 'uuid';
|
||||
import {
|
||||
megabyte,
|
||||
fileConfig,
|
||||
imageExtRegex,
|
||||
inferMimeType,
|
||||
mergeCodeEnvRef,
|
||||
} from 'librechat-data-provider';
|
||||
import { megabyte, fileConfig, imageExtRegex, inferMimeType } from 'librechat-data-provider';
|
||||
import type { CodeEnvRef, FileSources, getEndpointFileConfig } from 'librechat-data-provider';
|
||||
import type { CodeFileCommitData } from '@librechat/data-schemas';
|
||||
import type {
|
||||
|
|
@ -240,24 +234,8 @@ export async function processCodeOutput(
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ordinary outputs atomically reuse a file_id for this filename and conversation.
|
||||
* Explicit publications instead receive a fresh attempt ID from the gateway.
|
||||
*
|
||||
* Claim by `safeName` (not raw `name`) so the claim and the eventual
|
||||
* `createFile` agree on the filename column — otherwise weird inputs
|
||||
* (e.g. `"proj name/file@v1.txt"`) would claim under the raw name and
|
||||
* then write under the sanitized one, leaving the claim row orphaned.
|
||||
*/
|
||||
/**
|
||||
* Dispatch-order stamp persisted with every write AND every claim insert
|
||||
* (foreground writes dispatch ≈ now): the out-of-order guard below
|
||||
* compares WRITER dispatch order, not wall-clock write time — an older
|
||||
* task writing late must not make a newer task's harvest look stale, and
|
||||
* a freshly claimed row must carry its claimant's stamp before the
|
||||
* content write lands.
|
||||
*/
|
||||
const sourceDispatchedAt = freshClaimAfter ?? Date.now();
|
||||
// Separate captures never share primary storage, even at the same sandbox path.
|
||||
const sourceDispatchedAt = freshClaimAfter ?? currentDate.getTime();
|
||||
|
||||
const outputPersistence = createCodeOutputPersistence({
|
||||
publication,
|
||||
|
|
@ -272,36 +250,9 @@ export async function processCodeOutput(
|
|||
user: req.user.id,
|
||||
tenantId: req.user.tenantId,
|
||||
sourceDispatchedAt,
|
||||
snapshot: publication == null,
|
||||
});
|
||||
const file_id = claimed.file_id;
|
||||
const isUpdate = file_id !== newFileId;
|
||||
|
||||
/**
|
||||
* Out-of-order guard for detached (background) harvests: when the claimed
|
||||
* row's last writer was dispatched AFTER this task (`freshClaimAfter` =
|
||||
* this task's dispatch time), a newer run owns this filename slot. The
|
||||
* `(filename, conversationId)` unique index means the stale bytes have
|
||||
* nowhere else to live, so skip this file rather than overwrite fresh
|
||||
* content — the harvest's stdout patch still lands, only the superseded
|
||||
* attachment is omitted. Falls back to `updatedAt` for rows written
|
||||
* before the stamp existed (the claim itself is timestamp-neutral).
|
||||
*/
|
||||
const lastWriterDispatchedAt =
|
||||
claimed.metadata?.sourceDispatchedAt ??
|
||||
(claimed.updatedAt != null ? new Date(claimed.updatedAt).getTime() : null);
|
||||
if (isUpdate && freshClaimAfter != null && (lastWriterDispatchedAt ?? 0) > freshClaimAfter) {
|
||||
logger.warn(
|
||||
`[processCodeOutput] Skipping stale background output "${safeName}" (${file_id}): a newer run owns this filename`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isUpdate) {
|
||||
logger.debug(
|
||||
`[processCodeOutput] Updating existing file "${safeName}" (${file_id}) instead of creating duplicate`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Retain the completed output for token-qualified persistence after storage settles. */
|
||||
let pendingOutput: CodeOutputStoredFile | undefined;
|
||||
async function commitCodeFile(fileData: CodeOutputStoredFile): Promise<boolean> {
|
||||
|
|
@ -312,29 +263,18 @@ export async function processCodeOutput(
|
|||
const writeOutput = async (
|
||||
onUploadStart: FileStorageUploadObserver,
|
||||
): Promise<CodeOutputResult | null> => {
|
||||
/**
|
||||
* Preserve the original `messageId` on update. Each `processCodeOutput`
|
||||
* call would otherwise overwrite it with the current run's run id, which
|
||||
* decouples the file from the assistant message that originally created
|
||||
* it. `getCodeGeneratedFiles` filters by `messageId IN <thread>`, so a
|
||||
* stale id (e.g. from a later regeneration / failed re-read attempt)
|
||||
* silently excludes the file from priming on subsequent turns.
|
||||
*/
|
||||
const persistedMessageId = isUpdate ? (claimed.messageId ?? messageId) : messageId;
|
||||
/* A generated-output write replaces the file's bytes, so pointers to
|
||||
* earlier content in another profile must not survive as reusable refs. */
|
||||
const codeEnvReferenceSet = mergeCodeEnvRef(undefined, codeEnvRef);
|
||||
// The sandbox object is a live path, not an owned immutable copy.
|
||||
const codeEnvMetadata = {
|
||||
...claimed.metadata,
|
||||
...codeEnvReferenceSet,
|
||||
codeEnvSource: codeEnvRef,
|
||||
codeDestination: true,
|
||||
sourceDispatchedAt,
|
||||
};
|
||||
|
||||
if (isImage) {
|
||||
const usage = isUpdate ? (claimed.usage ?? 0) + 1 : 1;
|
||||
const usage = 1;
|
||||
onUploadStart();
|
||||
const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
|
||||
const filepath = usage > 1 ? `${_file.filepath}?v=${Date.now()}` : _file.filepath;
|
||||
const filepath = _file.filepath;
|
||||
const storageMetadata = getStorageMetadata({
|
||||
filepath: _file.filepath,
|
||||
source: appConfig.fileStrategy,
|
||||
|
|
@ -358,7 +298,7 @@ export async function processCodeOutput(
|
|||
filepath,
|
||||
...storageMetadata,
|
||||
file_id,
|
||||
messageId: persistedMessageId,
|
||||
messageId,
|
||||
usage,
|
||||
filename: safeName,
|
||||
conversationId,
|
||||
|
|
@ -366,7 +306,7 @@ export async function processCodeOutput(
|
|||
user: req.user.id,
|
||||
tenantId: req.user.tenantId,
|
||||
type: `image/${appConfig.imageOutputType}`,
|
||||
createdAt: isUpdate ? claimed.createdAt : formattedDate,
|
||||
createdAt: formattedDate,
|
||||
updatedAt: formattedDate,
|
||||
source: appConfig.fileStrategy,
|
||||
context: outputPersistence.context,
|
||||
|
|
@ -478,7 +418,7 @@ export async function processCodeOutput(
|
|||
file_id,
|
||||
filepath,
|
||||
...storageMetadata,
|
||||
messageId: persistedMessageId,
|
||||
messageId,
|
||||
object: 'file',
|
||||
filename: safeName,
|
||||
type: mimeType,
|
||||
|
|
@ -490,8 +430,8 @@ export async function processCodeOutput(
|
|||
metadata: codeEnvMetadata,
|
||||
source: appConfig.fileStrategy,
|
||||
context: outputPersistence.context,
|
||||
usage: isUpdate ? (claimed.usage ?? 0) + 1 : 1,
|
||||
createdAt: isUpdate ? claimed.createdAt : formattedDate,
|
||||
usage: 1,
|
||||
createdAt: formattedDate,
|
||||
...(await retentionExpiryPromise),
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ describe('Code output publication gateway', () => {
|
|||
expect(commit).toHaveBeenCalledWith(file);
|
||||
const finalize = async () => file;
|
||||
expect(persistence.finalize(file, finalize)).toBe(finalize);
|
||||
expect(persistence.context).toBe(FileContext.execute_code);
|
||||
expect(persistence.context).toBe(FileContext.execute_code_snapshot);
|
||||
});
|
||||
|
||||
it('uses a unique attempt ID until publication and then returns the canonical file identity', async () => {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ interface CodeOutputClaimInput {
|
|||
user: string;
|
||||
tenantId?: string | null;
|
||||
sourceDispatchedAt?: number;
|
||||
snapshot?: boolean;
|
||||
}
|
||||
|
||||
interface CodeOutputClaim {
|
||||
|
|
@ -103,7 +104,7 @@ export function createCodeOutputPersistence({
|
|||
}): CodeOutputPersistence {
|
||||
if (publication == null) {
|
||||
return {
|
||||
context: FileContext.execute_code,
|
||||
context: FileContext.execute_code_snapshot,
|
||||
claim,
|
||||
commit,
|
||||
trackStored: () => undefined,
|
||||
|
|
|
|||
|
|
@ -81,6 +81,44 @@ describe('queued code-file destinations', () => {
|
|||
expect(child.provisionState?.codeEnvDestinations?.get('shared')).toBe('data.csv');
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'keeps one shared snapshot name across private contributors (reverse %s)',
|
||||
(reverse) => {
|
||||
const shared = file('shared', { context: FileContext.execute_code_snapshot });
|
||||
const privateFile = file('private', {
|
||||
metadata: {
|
||||
codeEnvRef: {
|
||||
kind: 'agent',
|
||||
id: 'child',
|
||||
storage_session_id: 'private-session',
|
||||
file_id: 'private-object',
|
||||
sandboxFilename: 'data.csv',
|
||||
},
|
||||
},
|
||||
});
|
||||
const parent = agent('parent', [shared]);
|
||||
const child = agent('child', [shared, privateFile]);
|
||||
child.provisionState!.agentScopedFileIds.add('private');
|
||||
const contexts = reverse ? [child, parent] : [parent, child];
|
||||
for (const context of contexts) prepareQueuedCodeFileContext(context, contexts, 'user-1');
|
||||
for (const context of contexts) {
|
||||
const uploads = planCodeFileUploads({
|
||||
context,
|
||||
contexts,
|
||||
agentId: context.id,
|
||||
userId: 'user-1',
|
||||
useAdvertisedNames: true,
|
||||
});
|
||||
expect(uploads.find(({ file }) => file.file_id === 'shared')?.destination).toBe('data.csv');
|
||||
expect(new Set(uploads.map(({ destination }) => destination)).size).toBe(uploads.length);
|
||||
}
|
||||
expect(child.provisionState?.codeEnvDestinations?.get('private')).toMatch(/^data-.+\.csv$/);
|
||||
expect(child.dynamicToolContextMap?.queued_code_files).toContain(
|
||||
child.provisionState?.codeEnvDestinations?.get('private'),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('retains advertised paths after one file succeeds and another remains queued', () => {
|
||||
const current = agent('parent', [file('first'), file('second')]);
|
||||
prepareQueuedCodeFileContext(current, [current], 'user-1');
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import { getCodeEnvRefForProfile, resolveSandboxFilename } from 'librechat-data-provider';
|
||||
import {
|
||||
FileContext,
|
||||
getCodeEnvRefForProfile,
|
||||
resolveSandboxFilename,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TFile, TurnFileConsumers, AgentToolResources } from 'librechat-data-provider';
|
||||
import type { CodeEnvFile } from '@librechat/agents';
|
||||
import type { CodeExecutionRoute } from '../provision/service';
|
||||
|
|
@ -9,6 +13,9 @@ import {
|
|||
createCodeDestinationSet,
|
||||
reserveCodeDestination,
|
||||
sortCodeFilesByDestinationPriority,
|
||||
selectCodeFileGenerations,
|
||||
getCodeFileDestination,
|
||||
planCodeSnapshotMounts,
|
||||
} from './destinations';
|
||||
import { appendCodeFileContextLine, getCodeFileContextLine, getCodeFileLocation } from './priming';
|
||||
import { getCodeEnvUploadFilename } from './form';
|
||||
|
|
@ -79,11 +86,13 @@ export function planCodeFileUploads({
|
|||
}
|
||||
|
||||
const confirmedNames = new Set<string>();
|
||||
const files = sortCodeFilesByDestinationPriority(
|
||||
const files = selectCodeFileGenerations(
|
||||
[...state.codeEnvFiles, ...liveFiles.filter((file) => !queuedIds.has(file.file_id))],
|
||||
privateIds,
|
||||
).filter((file): file is TFile => {
|
||||
route,
|
||||
).files.filter((file): file is TFile => {
|
||||
if (!file) return false;
|
||||
if (file.context === FileContext.execute_code_snapshot) return true;
|
||||
const ref = getCodeEnvRefForProfile(file.metadata, route);
|
||||
const recovery = state.codeEnvRecoveryNames?.get(file.file_id);
|
||||
const entityId = state.agentScopedFileIds.has(file.file_id) ? agentId : undefined;
|
||||
|
|
@ -99,17 +108,34 @@ export function planCodeFileUploads({
|
|||
});
|
||||
|
||||
const destinations = createCodeDestinationSet();
|
||||
const snapshotMounts = planCodeSnapshotMounts(
|
||||
[...queued.values()].map((candidate) => candidate.file).concat(liveFiles),
|
||||
privateIds,
|
||||
route,
|
||||
state.codeEnvDestinations,
|
||||
);
|
||||
for (const name of snapshotMounts.values()) reserveCodeDestination(destinations, name);
|
||||
const liveNames = new Map<string, string>();
|
||||
const reserveLiveFiles = (candidate: ProvisionToolContext): void => {
|
||||
const candidateQueuedIds = new Set(
|
||||
candidate.provisionState?.codeEnvFiles.map((file) => file.file_id),
|
||||
);
|
||||
for (const file of candidate.tool_resources?.execute_code?.files ?? []) {
|
||||
if (candidateQueuedIds.has(file.file_id)) continue;
|
||||
const live = candidate.tool_resources?.execute_code?.files ?? [];
|
||||
const winners = new Set(
|
||||
selectCodeFileGenerations(
|
||||
[...(candidate.provisionState?.codeEnvFiles ?? []), ...live],
|
||||
privateIds,
|
||||
route,
|
||||
).files.flatMap((file) => (file ? [file.file_id] : [])),
|
||||
);
|
||||
for (const file of live) {
|
||||
if (candidateQueuedIds.has(file.file_id) || !winners.has(file.file_id)) continue;
|
||||
const name =
|
||||
getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
|
||||
candidate.provisionState?.codeEnvDestinations?.get(file.file_id) ??
|
||||
resolveSandboxFilename(file.filename, file.type);
|
||||
file.context === FileContext.execute_code_snapshot
|
||||
? getCodeFileDestination(file, route)
|
||||
: (getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
|
||||
candidate.provisionState?.codeEnvDestinations?.get(file.file_id) ??
|
||||
resolveSandboxFilename(file.filename, file.type));
|
||||
if (reserveCodeDestination(destinations, name) && !liveNames.has(file.file_id)) {
|
||||
liveNames.set(file.file_id, name);
|
||||
}
|
||||
|
|
@ -131,8 +157,9 @@ export function planCodeFileUploads({
|
|||
// An agent's frozen plan survives later discovery in other execution contexts.
|
||||
for (const [id, name] of state.codeEnvDestinations ?? []) {
|
||||
if (!queued.has(id)) continue;
|
||||
reserveCodeDestination(destinations, name);
|
||||
advertisedNames.set(id, name);
|
||||
const destination = snapshotMounts.get(id) ?? name;
|
||||
reserveCodeDestination(destinations, destination);
|
||||
advertisedNames.set(id, destination);
|
||||
}
|
||||
// Inherited paths are hints until checked against this agent's live mounts.
|
||||
for (const [id, candidate] of queued) {
|
||||
|
|
@ -152,15 +179,18 @@ export function planCodeFileUploads({
|
|||
if (!file) continue;
|
||||
const candidate = queued.get(file.file_id);
|
||||
const destination =
|
||||
snapshotMounts.get(file.file_id) ??
|
||||
advertisedNames.get(file.file_id) ??
|
||||
liveNames.get(file.file_id) ??
|
||||
claimCodeDestination(
|
||||
destinations,
|
||||
getCodeEnvUploadFilename(
|
||||
resolveSandboxFilename(
|
||||
getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
|
||||
candidate?.recoveryName ??
|
||||
file.filename,
|
||||
file.context === FileContext.execute_code_snapshot
|
||||
? getCodeFileDestination(file, route)
|
||||
: (getCodeEnvRefForProfile(file.metadata, route)?.sandboxFilename ??
|
||||
candidate?.recoveryName ??
|
||||
file.filename),
|
||||
file.type,
|
||||
),
|
||||
),
|
||||
|
|
|
|||
15
packages/api/src/files/consumers.ts
Normal file
15
packages/api/src/files/consumers.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import { hasToolResourceDestination } from 'librechat-data-provider';
|
||||
import type { EToolResources, TFile } from 'librechat-data-provider';
|
||||
|
||||
/** Chooser declines remain declines; cleanup does not erase destination intent. */
|
||||
export function allowsFileResource(
|
||||
file: Partial<Pick<TFile, 'file_id' | 'metadata' | 'embedded'>>,
|
||||
resource: EToolResources,
|
||||
membership?: ReadonlyMap<EToolResources, ReadonlySet<string>>,
|
||||
): boolean {
|
||||
return (
|
||||
file.metadata?.destinationChosen === false ||
|
||||
membership?.get(resource)?.has(file.file_id ?? '') === true ||
|
||||
hasToolResourceDestination(file, resource)
|
||||
);
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ export enum FileContext {
|
|||
agents = 'agents',
|
||||
assistants = 'assistants',
|
||||
execute_code = 'execute_code',
|
||||
execute_code_snapshot = 'execute_code_snapshot',
|
||||
image_generation = 'image_generation',
|
||||
assistants_output = 'assistants_output',
|
||||
message_attachment = 'message_attachment',
|
||||
|
|
@ -230,6 +231,8 @@ export type TFile = {
|
|||
*/
|
||||
codeEnvRef?: CodeEnvRef;
|
||||
codeEnvRefs?: CodeEnvRefMap;
|
||||
/** Non-owning provenance. Never reused or deleted as a stored copy. */
|
||||
codeEnvSource?: CodeEnvRef;
|
||||
/** Dispatch-order stamp for the current source artifact generation. */
|
||||
sourceDispatchedAt?: number;
|
||||
/** Vector namespaces this file has been embedded into. */
|
||||
|
|
|
|||
|
|
@ -451,6 +451,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
user: string;
|
||||
tenantId?: string | null;
|
||||
sourceDispatchedAt?: number;
|
||||
snapshot?: boolean;
|
||||
}) => Promise<IMongoFile>;
|
||||
commitCodeFile: (data: CodeFileCommitData, sourceDispatchedAt?: number) => Promise<boolean>;
|
||||
createFile: (data: Partial<IMongoFile>, disableTTL?: boolean) => Promise<IMongoFile | null>;
|
||||
|
|
@ -908,7 +909,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
const filter = withOwnerScope(
|
||||
{
|
||||
file_id: { $in: fileIds },
|
||||
context: { $ne: FileContext.execute_code },
|
||||
context: { $nin: [FileContext.execute_code, FileContext.execute_code_snapshot] },
|
||||
$or: orConditions,
|
||||
},
|
||||
ownerScope,
|
||||
|
|
@ -982,7 +983,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
const filter = withOwnerScope(
|
||||
{
|
||||
conversationId,
|
||||
context: FileContext.execute_code,
|
||||
context: { $in: [FileContext.execute_code, FileContext.execute_code_snapshot] },
|
||||
file_id: { $in: threadFileIds },
|
||||
$or: [
|
||||
{ 'metadata.codeEnvRef': { $exists: true } },
|
||||
|
|
@ -1113,7 +1114,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
const filter = withOwnerScope(
|
||||
{
|
||||
file_id: { $in: fileIds },
|
||||
context: { $ne: FileContext.execute_code },
|
||||
context: { $nin: [FileContext.execute_code, FileContext.execute_code_snapshot] },
|
||||
...(hydrateEverything ? {} : { $or: missingConditions }),
|
||||
},
|
||||
ownerScope,
|
||||
|
|
@ -1154,7 +1155,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
const filter = withOwnerScope(
|
||||
{
|
||||
file_id: { $in: fileIds },
|
||||
context: { $ne: FileContext.execute_code },
|
||||
context: { $nin: [FileContext.execute_code, FileContext.execute_code_snapshot] },
|
||||
$or: [
|
||||
{ 'metadata.codeEnvRef': { $exists: true } },
|
||||
{ 'metadata.codeEnvRefs': { $exists: true } },
|
||||
|
|
@ -1190,6 +1191,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
* freshly claimed (not-yet-written) row still carries an ownership
|
||||
* signal for the background harvest's stale-output guard. */
|
||||
sourceDispatchedAt?: number;
|
||||
snapshot?: boolean;
|
||||
}): Promise<IMongoFile> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const tenantFilter = data.tenantId ? { tenantId: data.tenantId } : { tenantId: null };
|
||||
|
|
@ -1199,6 +1201,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
file_id: data.file_id,
|
||||
user: data.user,
|
||||
...(data.tenantId ? { tenantId: data.tenantId } : {}),
|
||||
...(data.snapshot ? { expiresAt: new Date(Date.now() + 3600 * 1000) } : {}),
|
||||
...(data.sourceDispatchedAt != null
|
||||
? { metadata: { sourceDispatchedAt: data.sourceDispatchedAt } }
|
||||
: {}),
|
||||
|
|
@ -1209,7 +1212,8 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
{
|
||||
filename: data.filename,
|
||||
conversationId: data.conversationId,
|
||||
context: FileContext.execute_code,
|
||||
context: data.snapshot ? FileContext.execute_code_snapshot : FileContext.execute_code,
|
||||
...(data.snapshot ? { file_id: data.file_id, user: data.user } : {}),
|
||||
...tenantFilter,
|
||||
...codeFileStorageFilter(),
|
||||
},
|
||||
|
|
@ -1251,7 +1255,9 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
|
|||
delete fileData.expiresAt;
|
||||
}
|
||||
|
||||
const generatedOutput = data.context === FileContext.execute_code;
|
||||
const generatedOutput =
|
||||
data.context === FileContext.execute_code ||
|
||||
data.context === FileContext.execute_code_snapshot;
|
||||
try {
|
||||
return await File.findOneAndUpdate(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { FileContext } from 'librechat-data-provider';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import type { FileStorageClaim, IMongoFile } from '~/types/file';
|
||||
import type { AllMethods, FileStorageMethods } from './index';
|
||||
import { _resetStrictCache } from '~/models/plugins/tenantIsolation';
|
||||
import { tenantStorage, runAsSystem } from '~/config/tenantContext';
|
||||
import { createFileStorageMethods } from './fileStorage';
|
||||
import { createMethods } from './index';
|
||||
import { createModels } from '~/models';
|
||||
|
||||
let File: mongoose.Model<IMongoFile>;
|
||||
|
|
@ -71,6 +73,179 @@ describe('file storage fence', () => {
|
|||
const publicMethods: Pick<AllMethods, keyof FileStorageMethods> = methods;
|
||||
expect(Object.keys(publicMethods)).toHaveLength(5);
|
||||
});
|
||||
describe('unfinished snapshot releases', () => {
|
||||
async function reserve() {
|
||||
return createMethods(mongoose).claimCodeFile({
|
||||
filename: 'report.txt',
|
||||
conversationId: 'snapshot-conversation',
|
||||
file_id: claim.file_id,
|
||||
user: claim.userId,
|
||||
tenantId: claim.tenantId,
|
||||
sourceDispatchedAt: 100,
|
||||
snapshot: true,
|
||||
});
|
||||
}
|
||||
|
||||
it('gives only a fresh snapshot reservation the existing upload TTL', async () => {
|
||||
const startedAt = Date.now();
|
||||
await reserve();
|
||||
const reservation = await read();
|
||||
expect(reservation?.expiresAt).toBeInstanceOf(Date);
|
||||
expect(reservation!.expiresAt!.getTime()).toBeGreaterThanOrEqual(startedAt + 3_600_000);
|
||||
expect(reservation!.expiresAt!.getTime()).toBeLessThanOrEqual(Date.now() + 3_600_000);
|
||||
expect(reservation?.filepath).toBeUndefined();
|
||||
expect(reservation?.storageOperations?.provisioning).toBeUndefined();
|
||||
await File.init();
|
||||
expect(await File.collection.indexes()).toContainEqual(
|
||||
expect.objectContaining({ key: { expiresAt: 1 }, expireAfterSeconds: 3600 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('retains the deadline when acquisition fails before the database write', async () => {
|
||||
await reserve();
|
||||
const reservation = await read();
|
||||
expect(reservation?.expiresAt).toBeInstanceOf(Date);
|
||||
jest.spyOn(File, 'updateOne').mockRejectedValueOnce(new Error('Database unavailable'));
|
||||
await expect(peer.acquireFileProvision(claim)).rejects.toThrow('Database unavailable');
|
||||
expect(await read()).toEqual(reservation);
|
||||
expect((await read())?.storageOperations?.provisioning).toBeUndefined();
|
||||
});
|
||||
|
||||
it('retains the deadline when another scope cannot acquire the reservation', async () => {
|
||||
await reserve();
|
||||
const reservation = await read();
|
||||
expect(reservation?.expiresAt).toBeInstanceOf(Date);
|
||||
expect(await peer.acquireFileProvision({ ...claim, tenantId: 'tenant-b' })).toBe(false);
|
||||
expect(await read()).toEqual(reservation);
|
||||
});
|
||||
|
||||
it('clears the deadline atomically even when the acquisition acknowledgement is lost', async () => {
|
||||
await reserve();
|
||||
expect((await read())?.expiresAt).toBeInstanceOf(Date);
|
||||
jest.spyOn(peer, 'acquireFileProvision').mockImplementationOnce(async (data) => {
|
||||
expect(await methods.acquireFileProvision(data)).toBe(true);
|
||||
throw new Error('Acquisition acknowledgement lost');
|
||||
});
|
||||
await expect(peer.acquireFileProvision(claim)).rejects.toThrow(
|
||||
'Acquisition acknowledgement lost',
|
||||
);
|
||||
const acquired = await read();
|
||||
expect(acquired?.expiresAt).toBeUndefined();
|
||||
expect(acquired?.storageOperations?.provisioning).toEqual([claim.token]);
|
||||
});
|
||||
|
||||
it('does not extend an existing reservation deadline', async () => {
|
||||
await reserve();
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
await File.collection.updateOne(
|
||||
{ file_id: claim.file_id },
|
||||
{ $set: { expiresAt: deadline } },
|
||||
);
|
||||
const reservation = await read();
|
||||
await reserve();
|
||||
expect(await read()).toEqual(reservation);
|
||||
});
|
||||
|
||||
it('does not add a deadline to an existing committed snapshot', async () => {
|
||||
await reserve();
|
||||
await methods.acquireFileProvision(claim);
|
||||
await File.collection.updateOne(
|
||||
{ file_id: claim.file_id },
|
||||
{ $set: { filepath: '/uploads/committed', bytes: 100 } },
|
||||
);
|
||||
await methods.releaseFileProvision(claim);
|
||||
const committed = await read();
|
||||
expect(committed?.expiresAt).toBeUndefined();
|
||||
await reserve();
|
||||
expect(await read()).toEqual(committed);
|
||||
});
|
||||
|
||||
it('leaves legacy claims without a provisional deadline', async () => {
|
||||
const legacy = await createMethods(mongoose).claimCodeFile({
|
||||
filename: 'legacy.txt',
|
||||
conversationId: 'legacy-conversation',
|
||||
file_id: 'legacy-file',
|
||||
user: claim.userId,
|
||||
tenantId: claim.tenantId,
|
||||
});
|
||||
expect(legacy.context).toBe(FileContext.execute_code);
|
||||
expect(legacy.expiresAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it('deletes only the unfinished claim and accepts repeated releases', async () => {
|
||||
await reserve();
|
||||
expect((await read())?.object).toBe('file');
|
||||
expect(await methods.acquireFileProvision(claim)).toBe(true);
|
||||
expect(await methods.commitFileProvision({ claim, releaseOnly: true })).toBe(true);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await read()).toBeNull();
|
||||
expect(await peer.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await methods.acquireFileProvision(claim)).toBe(false);
|
||||
expect(await methods.commitFileProvision({ claim, releaseOnly: true })).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ userId: new mongoose.Types.ObjectId().toString() },
|
||||
{ tenantId: 'tenant-b' },
|
||||
{ tenantId: null },
|
||||
{ token: 'foreign' },
|
||||
])('does not discard another scope or token: %j', async (overrides) => {
|
||||
await reserve();
|
||||
await methods.acquireFileProvision(claim);
|
||||
const before = await read();
|
||||
expect(await runAsSystem(() => peer.releaseFileProvision({ ...claim, ...overrides }))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(await read()).toEqual(before);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await read()).toBeNull();
|
||||
});
|
||||
|
||||
it('retains other claimants and deletes on the final concurrent release', async () => {
|
||||
for (let round = 0; round < 4; round++) {
|
||||
await reserve();
|
||||
const claims = Array.from({ length: 16 }, (_, i) => ({ ...claim, token: `token-${i}` }));
|
||||
expect(await Promise.all(claims.map(methods.acquireFileProvision))).toEqual(
|
||||
claims.map(() => true),
|
||||
);
|
||||
expect(await methods.releaseFileProvision(claims[0])).toBe(true);
|
||||
expect((await read())?.storageOperations?.provisioning).toHaveLength(15);
|
||||
expect(await Promise.all(claims.slice(1).map(peer.releaseFileProvision))).toEqual(
|
||||
claims.slice(1).map(() => true),
|
||||
);
|
||||
expect(await read()).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ filepath: '/uploads/committed' },
|
||||
{ bytes: 0 },
|
||||
{ storageKey: 'primary-object' },
|
||||
{ context: FileContext.execute_code },
|
||||
{ context: FileContext.message_attachment },
|
||||
{ 'storageOperations.deletionPending': true },
|
||||
{ 'storageOperations.deleting': 'deletion-owner' },
|
||||
])('preserves committed content, other contexts and deletion intent: %j', async (fields) => {
|
||||
await reserve();
|
||||
await methods.acquireFileProvision(claim);
|
||||
await File.collection.updateOne({ file_id: claim.file_id }, { $set: fields });
|
||||
const before = await read();
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await read()).toEqual({
|
||||
...before,
|
||||
storageOperations: { ...before?.storageOperations, provisioning: [] },
|
||||
});
|
||||
});
|
||||
|
||||
it('retains committed snapshot metadata and primary content', async () => {
|
||||
await seed({ context: FileContext.execute_code_snapshot });
|
||||
const before = await read();
|
||||
await methods.acquireFileProvision(claim);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await read()).toEqual({ ...before, storageOperations: { provisioning: [] } });
|
||||
});
|
||||
});
|
||||
|
||||
it('retains every concurrent provisioning owner until release', async () => {
|
||||
await seed();
|
||||
const claims = ['code-a', 'code-b', 'search'].map((token) => ({ ...claim, token }));
|
||||
|
|
@ -232,7 +407,7 @@ describe('file storage fence', () => {
|
|||
|
||||
it('never upserts missing files for any claim or release', async () => {
|
||||
expect(await methods.acquireFileProvision(claim)).toBe(false);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(false);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await methods.acquireFileDeletion(deletion)).toBeNull();
|
||||
expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(false);
|
||||
expect(await File.collection.countDocuments()).toBe(0);
|
||||
|
|
@ -250,7 +425,7 @@ describe('file storage fence', () => {
|
|||
expect(await runAsSystem(() => peer.acquireFileProvision(other))).toBe(false);
|
||||
expect(await runAsSystem(() => peer.acquireFileDeletion({ ...other, updatedAt }))).toBeNull();
|
||||
expect(await methods.acquireFileProvision(claim)).toBe(true);
|
||||
expect(await runAsSystem(() => peer.releaseFileProvision(other))).toBe(false);
|
||||
expect(await runAsSystem(() => peer.releaseFileProvision(other))).toBe(true);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
|
||||
expect(
|
||||
|
|
@ -289,7 +464,7 @@ describe('file storage fence', () => {
|
|||
await tenantStorage.run(context, () => methods.acquireFileDeletion(deletion)),
|
||||
).toBeNull();
|
||||
expect(await methods.acquireFileProvision(claim)).toBe(true);
|
||||
expect(await tenantStorage.run(context, () => methods.releaseFileProvision(claim))).toBe(false);
|
||||
expect(await tenantStorage.run(context, () => methods.releaseFileProvision(claim))).toBe(true);
|
||||
expect(await methods.releaseFileProvision(claim)).toBe(true);
|
||||
expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { FileContext } from 'librechat-data-provider';
|
||||
import type { FilterQuery, Model } from 'mongoose';
|
||||
import type {
|
||||
FileStorageClaim,
|
||||
|
|
@ -50,15 +51,32 @@ export function createFileStorageMethods(mongoose: typeof import('mongoose')): {
|
|||
return result.matchedCount === 1;
|
||||
}
|
||||
|
||||
/** An already-applied release is successful; only the supplied token is removed. */
|
||||
/** Safe releases also discard unfinished snapshots when the last claimant leaves. */
|
||||
async function releaseFileProvision(data: FileStorageClaim): Promise<boolean> {
|
||||
const File = mongoose.models.File as Model<IMongoFile>;
|
||||
const result = await File.updateOne(
|
||||
claimFilter(data),
|
||||
{ $pull: { 'storageOperations.provisioning': data.token } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
return result.matchedCount === 1;
|
||||
const scope = claimFilter(data);
|
||||
const disposable: FilterQuery<IMongoFile> = {
|
||||
context: FileContext.execute_code_snapshot,
|
||||
filepath: { $exists: false },
|
||||
bytes: { $exists: false },
|
||||
storageKey: { $exists: false },
|
||||
'storageOperations.provisioning': [data.token],
|
||||
'storageOperations.deletionPending': { $ne: true },
|
||||
'storageOperations.deleting': { $exists: false },
|
||||
};
|
||||
for (;;) {
|
||||
const deleted = await File.deleteOne({ ...scope, ...disposable });
|
||||
if (deleted.deletedCount === 1) return true;
|
||||
const result = await File.updateOne(
|
||||
// A concurrent release can leave us as the last claimant after the delete missed.
|
||||
{ ...scope, $nor: [disposable] },
|
||||
{ $pull: { 'storageOperations.provisioning': data.token } },
|
||||
{ timestamps: false },
|
||||
);
|
||||
if (result.matchedCount === 1) return true;
|
||||
// A lost delete acknowledgement must be safe to retry without recreating the row.
|
||||
if ((await File.exists(scope)) == null) return true;
|
||||
}
|
||||
}
|
||||
|
||||
async function commitFileProvision(data: FileProvisionCommit): Promise<boolean> {
|
||||
|
|
|
|||
|
|
@ -162,6 +162,10 @@ const file: Schema<IMongoFile> = new Schema(
|
|||
type: codeEnvRefMapSchema,
|
||||
default: undefined,
|
||||
},
|
||||
codeEnvSource: {
|
||||
type: codeEnvRefSchema,
|
||||
default: undefined,
|
||||
},
|
||||
/** Dispatch-order stamp of the last writer (or claimant, on insert):
|
||||
* the background harvest's stale-output guard compares writer
|
||||
* dispatch order so an older task settling late cannot overwrite a
|
||||
|
|
|
|||
|
|
@ -221,6 +221,8 @@ export interface IMongoFile extends Omit<Document, 'model'> {
|
|||
*/
|
||||
codeEnvRef?: CodeEnvRef;
|
||||
codeEnvRefs?: CodeEnvRefMap;
|
||||
/** Non-owning provenance of an immutable generated snapshot. */
|
||||
codeEnvSource?: CodeEnvRef;
|
||||
/** Dispatch-order stamp for the current source artifact generation. */
|
||||
sourceDispatchedAt?: number;
|
||||
/** Vector namespaces this file has been embedded into. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue