mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 14:33:38 +00:00
📭 fix: Deliver Attachment Text Until a File Tool Holds It (#16027)
* 📭 fix: Deliver Attachment Text Until a File Tool Holds It
A `none`-routed upload had its extracted text withheld whenever a file tool was
enabled, on the assumption the tool would serve the file. A plain chat with the
File Search toggle runs an ephemeral agent, so the upload is filed under no tool
resource and never embedded: the text was withheld for a vector store that never
received the file, and the attachment reached nothing.
Withholding now requires the record to show the tool holds the file, which is the
same evidence deferred provisioning reads before queueing it. `resources.ts`
delegates to that shared predicate so the two cannot disagree again.
* test: Give the Code-Running Primary Agent a Sandbox Copy
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
c153c4057f
commit
9e83b10545
8 changed files with 359 additions and 35 deletions
|
|
@ -2661,14 +2661,30 @@ describe('BaseClient', () => {
|
|||
metadata: { destinationChosen: false },
|
||||
};
|
||||
|
||||
/** A tool serves a file only once it holds it, so a record left to the sandbox needs the
|
||||
* reference provisioning writes for that tool to count as its reader. */
|
||||
const sandboxCsv = {
|
||||
...routedCsv,
|
||||
metadata: {
|
||||
destinationChosen: false,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
storage_session_id: 'session-1',
|
||||
file_id: 'sandbox-csv-file',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const replayCsv = async (
|
||||
fileConsumers,
|
||||
endpointConfig = {
|
||||
defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } },
|
||||
textFallbackWithoutTools: true,
|
||||
},
|
||||
file = routedCsv,
|
||||
) => {
|
||||
getFiles.mockResolvedValueOnce([routedCsv]);
|
||||
getFiles.mockResolvedValueOnce([file]);
|
||||
TestClient.options.req.config = {
|
||||
fileConfig: { endpoints: { [EModelEndpoint.openAI]: endpointConfig } },
|
||||
};
|
||||
|
|
@ -2743,13 +2759,30 @@ describe('BaseClient', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
test('keeps the file off the prompt when this turn can read it with code', async () => {
|
||||
const message = await replayCsv({ executeCode: true, fileSearch: false });
|
||||
test('keeps the file off the prompt when the sandbox running it holds the file', async () => {
|
||||
const message = await replayCsv(
|
||||
{ executeCode: true, fileSearch: false },
|
||||
undefined,
|
||||
sandboxCsv,
|
||||
);
|
||||
|
||||
expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([]);
|
||||
expect(TestClient.addFileContextToMessage).not.toHaveBeenCalled();
|
||||
expect(message.fileContext).toBeUndefined();
|
||||
});
|
||||
|
||||
test('replays the stored text when file search never received the file', async () => {
|
||||
/* An earlier turn's upload that named no destination was filed under no tool, so the
|
||||
* vector store holds nothing to search. Withholding the text for the enabled tool left
|
||||
* the file unreadable on every later turn as well as the one it arrived on. */
|
||||
const message = await replayCsv({ executeCode: false, fileSearch: true });
|
||||
const replayed = { ...routedCsv, llmDeliveryPath: 'text' };
|
||||
|
||||
expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([replayed]);
|
||||
expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [replayed]);
|
||||
expect(message.fileContext).toBe('region,total');
|
||||
expect(routedCsv.llmDeliveryPath).toBe('none');
|
||||
});
|
||||
});
|
||||
|
||||
test('rehydrates historical file refs from owner-scoped DB rows only', async () => {
|
||||
|
|
@ -3520,7 +3553,7 @@ describe('BaseClient', () => {
|
|||
});
|
||||
});
|
||||
|
||||
test('keeps a tool-routed file off the prompt when this turn can read it with code', () => {
|
||||
test('keeps a tool-routed file off the prompt when the sandbox running it holds the file', () => {
|
||||
routeCsvToTools();
|
||||
TestClient.options.agent = {
|
||||
provider: EModelEndpoint.openAI,
|
||||
|
|
@ -3536,13 +3569,51 @@ describe('BaseClient', () => {
|
|||
type: 'text/csv',
|
||||
text: 'region,total',
|
||||
llmDeliveryPath: 'none',
|
||||
metadata: { destinationChosen: false },
|
||||
metadata: {
|
||||
destinationChosen: false,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
storage_session_id: 'session-1',
|
||||
file_id: 'sandbox-code-csv',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(TestClient.getAttachmentDeliveryPath(file)).toBe('none');
|
||||
expect(TestClient.getTextContextAttachments([file])).toEqual([]);
|
||||
});
|
||||
|
||||
test('delivers the text when the tool that would read the file has yet to receive it', () => {
|
||||
/* An upload that named no destination is filed under no tool, so the enabled tool alone
|
||||
* cannot serve it and withholding the text left it readable by nothing. */
|
||||
routeCsvToTools();
|
||||
TestClient.options.agent = {
|
||||
provider: EModelEndpoint.openAI,
|
||||
fileConsumers: { executeCode: true, fileSearch: true },
|
||||
};
|
||||
TestClient.options.agent.deliveryRouting = resolveTurnDeliveryRouting({
|
||||
agent: TestClient.options.agent,
|
||||
config: TestClient.options.req?.config,
|
||||
});
|
||||
const file = {
|
||||
file_id: 'unprovisioned-csv',
|
||||
filename: 'sales.csv',
|
||||
type: 'text/csv',
|
||||
text: 'region,total',
|
||||
llmDeliveryPath: 'none',
|
||||
metadata: { destinationChosen: false },
|
||||
};
|
||||
|
||||
expect(TestClient.getAttachmentDeliveryPath(file)).toBe('text');
|
||||
/* The filter selects records by the route this turn resolves, so it returns the record as
|
||||
* stored; marking the copy admission reads is `resolveTurnAttachments`. */
|
||||
expect(TestClient.getTextContextAttachments([file])).toEqual([file]);
|
||||
expect(TestClient.resolveTurnAttachments([file])).toEqual([
|
||||
{ ...file, llmDeliveryPath: 'text' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('does not fall back on an endpoint that has not enabled it', () => {
|
||||
routeCsvToTools({ textFallbackWithoutTools: false });
|
||||
TestClient.options.agent = {
|
||||
|
|
|
|||
|
|
@ -6718,7 +6718,18 @@ describe('AgentClient - titleConvo', () => {
|
|||
...makeUploadedFile('fallback-file', 'sales.csv', 'text/csv'),
|
||||
text: 'handoff fallback content',
|
||||
llmDeliveryPath: 'none',
|
||||
metadata: { destinationChosen: false },
|
||||
/* The primary agent runs code, and a tool serves a file only once it holds it, so the
|
||||
* sandbox reference is what keeps the text out of the primary prompt while the handoff
|
||||
* agent, which runs no reader at all, still receives it. */
|
||||
metadata: {
|
||||
destinationChosen: false,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
storage_session_id: 'session-1',
|
||||
file_id: 'sandbox-fallback-file',
|
||||
},
|
||||
},
|
||||
};
|
||||
const { resolveTurnDeliveryRouting } = jest.requireActual('@librechat/api');
|
||||
client.options.req.config.fileConfig = {
|
||||
|
|
|
|||
|
|
@ -4593,6 +4593,23 @@ describe('initializeAgent tool-routed text fallback', () => {
|
|||
metadata: { destinationChosen: false },
|
||||
}) as IMongoFile;
|
||||
|
||||
/** A tool serves a file only once it holds it, so these carry the evidence provisioning
|
||||
* writes: vectors for file search, a sandbox pointer for code execution. */
|
||||
const embeddedCsv = () => ({ ...routedCsv(), embedded: true }) as IMongoFile;
|
||||
const sandboxCsv = () =>
|
||||
({
|
||||
...routedCsv(),
|
||||
metadata: {
|
||||
destinationChosen: false,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user_1',
|
||||
storage_session_id: 'session_1',
|
||||
file_id: 'sandbox_file_1',
|
||||
},
|
||||
},
|
||||
}) as IMongoFile;
|
||||
|
||||
async function initializeWith({
|
||||
tools,
|
||||
csv,
|
||||
|
|
@ -4701,13 +4718,24 @@ describe('initializeAgent tool-routed text fallback', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it('keeps fallback out of the prompt when file search loads successfully', async () => {
|
||||
const csv = routedCsv();
|
||||
it('keeps fallback out of the prompt when file search loads and holds the file', async () => {
|
||||
const csv = embeddedCsv();
|
||||
const { result } = await initializeWith({ tools: [EToolResources.file_search], csv });
|
||||
expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true });
|
||||
expect(result.requestAttachments).toEqual([csv]);
|
||||
});
|
||||
|
||||
it('delivers fallback text when file search runs but never received the file', async () => {
|
||||
/* The plain-chat File Search toggle: the upload names no destination, so nothing files it
|
||||
* under a tool resource and the vector store stays empty. Withholding the text for the
|
||||
* toggle alone left the attachment readable by neither the model nor the tool. */
|
||||
const csv = routedCsv();
|
||||
const { result } = await initializeWith({ tools: [EToolResources.file_search], csv });
|
||||
expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true });
|
||||
expect(result.requestAttachments).toEqual([{ ...csv, llmDeliveryPath: 'text' }]);
|
||||
expect(csv.llmDeliveryPath).toBe('none');
|
||||
});
|
||||
|
||||
it('falls back after file search soft-fails and admits the returned text copy', async () => {
|
||||
const csv = routedCsv();
|
||||
const { result } = await initializeWith({
|
||||
|
|
@ -4828,8 +4856,8 @@ describe('initializeAgent tool-routed text fallback', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('leaves the file to Run Code when the agent can run code', async () => {
|
||||
const csv = routedCsv();
|
||||
it('leaves the file to Run Code when the sandbox already holds it', async () => {
|
||||
const csv = sandboxCsv();
|
||||
|
||||
const { result, filterFilesByEndpointRuntimeConfig } = await initializeWith({
|
||||
tools: [EToolResources.execute_code],
|
||||
|
|
|
|||
|
|
@ -36,6 +36,21 @@ const config = {
|
|||
const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false };
|
||||
const runsCode: TurnFileConsumers = { executeCode: true, fileSearch: false };
|
||||
|
||||
/** A tool serves a file only once it holds it, so a record left to the sandbox needs the
|
||||
* reference provisioning writes for that tool to count as its reader. */
|
||||
const inSandbox = <T extends { metadata?: Record<string, unknown> }>(file: T): T => ({
|
||||
...file,
|
||||
metadata: {
|
||||
...file.metadata,
|
||||
codeEnvRef: {
|
||||
kind: 'user' as const,
|
||||
id: 'user_1',
|
||||
storage_session_id: 'session_1',
|
||||
file_id: 'sandbox_file_1',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
describe('resolveTurnDeliveryRouting', () => {
|
||||
it('routes under the endpoint an agent names before its provider', () => {
|
||||
expect(resolveTurnDeliveryRouting({ agent: { provider: 'openAI' }, config }).endpoint).toBe(
|
||||
|
|
@ -97,12 +112,20 @@ describe('applyTurnDelivery', () => {
|
|||
expect(csv.llmDeliveryPath).toBe('none');
|
||||
});
|
||||
|
||||
it('returns the same array when a tool this turn runs can read every file', () => {
|
||||
const files = [csv, pdf];
|
||||
it('returns the same array when a tool this turn runs holds every file', () => {
|
||||
const files = [inSandbox(csv), pdf];
|
||||
|
||||
expect(applyTurnDelivery(files, { agent, config, consumers: runsCode })).toBe(files);
|
||||
});
|
||||
|
||||
it('delivers text for a file the tool that would read it has yet to receive', () => {
|
||||
/* An upload that named no destination is filed under no tool, so the enabled tool alone is
|
||||
* not what serves it: withholding the text on that basis left it readable by nothing. */
|
||||
expect(applyTurnDelivery([csv], { agent, config, consumers: runsCode })).toEqual([
|
||||
{ ...csv, llmDeliveryPath: 'text' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('marks nothing where the endpoint has not enabled the fallback', () => {
|
||||
const files = [csv];
|
||||
const disabled = {
|
||||
|
|
@ -199,10 +222,11 @@ describe('applyTurnDelivery', () => {
|
|||
|
||||
it('removes a record this turn leaves to tools from model admission', () => {
|
||||
/* Over-admitting cannot pass a limit; dropping a record the client still sends would. */
|
||||
const files = [{ ...csv, llmDeliveryPath: 'text' }];
|
||||
const held = inSandbox(csv);
|
||||
const files = [{ ...held, llmDeliveryPath: 'text' }];
|
||||
|
||||
expect(applyTurnDelivery(files, { agent, config, consumers: runsCode })).toEqual([
|
||||
{ ...csv, llmDeliveryPath: 'none' },
|
||||
{ ...held, llmDeliveryPath: 'none' },
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -184,14 +184,31 @@ describe('createRunFileMessageEncoder', () => {
|
|||
},
|
||||
});
|
||||
|
||||
const [noReader, runsCode, unknown] = await Promise.all([
|
||||
/* A tool serves a file only once it holds it, so the child that runs code keeps the file off
|
||||
* its prompt for the copy the sandbox has, and receives the text for the one it does not. */
|
||||
const sandboxCsv: TFile = {
|
||||
...csv,
|
||||
metadata: {
|
||||
destinationChosen: false,
|
||||
codeEnvRef: {
|
||||
kind: 'user',
|
||||
id: 'user-1',
|
||||
storage_session_id: 'session-1',
|
||||
file_id: 'sandbox-input-csv',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const [noReader, runsCode, unprovisioned, unknown] = await Promise.all([
|
||||
harness.encode([csv], 'noReader'),
|
||||
harness.encode([sandboxCsv], 'runsCode'),
|
||||
harness.encode([csv], 'runsCode'),
|
||||
harness.encode([csv], 'unknown'),
|
||||
]);
|
||||
|
||||
expect(JSON.stringify(noReader[0].content)).toContain('region,total');
|
||||
expect(runsCode).toEqual([]);
|
||||
expect(JSON.stringify(unprovisioned[0].content)).toContain('region,total');
|
||||
expect(unknown).toEqual([]);
|
||||
expect(harness.extractText).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ attachments: [{ ...csv, llmDeliveryPath: 'text' }] }),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
FileSources,
|
||||
getCodeEnvRefs,
|
||||
canToolResourceConsume,
|
||||
hasToolResourceProvisioning,
|
||||
} from 'librechat-data-provider';
|
||||
import type {
|
||||
AgentToolResources,
|
||||
|
|
@ -441,12 +442,10 @@ const computeProvisionState = async ({
|
|||
/** What the record itself shows about where it was sent. A message attachment is never
|
||||
* in the agent's resources, so a reference or an embedding is the only evidence that
|
||||
* the chooser picked that destination, and it is proof: nothing else creates one. */
|
||||
const carriesEvidenceFor = (file: TFile, resourceType: EToolResources): boolean => {
|
||||
if (resourceType === EToolResources.execute_code) {
|
||||
return getCodeEnvRefs(file.metadata).length > 0;
|
||||
}
|
||||
return file.embedded === true || (file.metadata?.embeddedEntities?.length ?? 0) > 0;
|
||||
};
|
||||
/* The predicate the turn's delivery decision reads too, so a file this queue has yet to
|
||||
* provision is never one the prompt withheld its text for. */
|
||||
const carriesEvidenceFor = (file: TFile, resourceType: EToolResources): boolean =>
|
||||
hasToolResourceProvisioning(file, resourceType);
|
||||
|
||||
const allowsResource = (file: TFile, resourceType: EToolResources): boolean => {
|
||||
if (!cameFromChooser(file)) {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import type { TEndpoint } from './config';
|
|||
import {
|
||||
hasTurnFileConsumer,
|
||||
isNativelyReadableText,
|
||||
hasToolResourceProvisioning,
|
||||
canToolResourceConsume,
|
||||
resolveUploadDestination,
|
||||
getCustomEndpointProvider,
|
||||
|
|
@ -19,6 +20,7 @@ import {
|
|||
SYSTEM_LLM_DELIVERY_DEFAULTS,
|
||||
} from './resolve-llm-delivery-path';
|
||||
import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config';
|
||||
import { EToolResources } from './types/tools';
|
||||
|
||||
function resolveTurnLLMDeliveryPath({
|
||||
file,
|
||||
|
|
@ -771,6 +773,15 @@ describe('provider document capability', () => {
|
|||
});
|
||||
});
|
||||
|
||||
const codeRef = {
|
||||
kind: 'user' as const,
|
||||
id: 'user_1',
|
||||
storage_session_id: 'session_1',
|
||||
file_id: 'sandbox_file_1',
|
||||
};
|
||||
/** File search reads an email export and the code interpreter's list does not offer it. */
|
||||
const eml = 'message/rfc822';
|
||||
|
||||
describe('hasTurnFileConsumer', () => {
|
||||
it('finds a reader only among the tools this turn runs', () => {
|
||||
expect(hasTurnFileConsumer('text/csv', { executeCode: false, fileSearch: false })).toBe(false);
|
||||
|
|
@ -781,6 +792,72 @@ describe('hasTurnFileConsumer', () => {
|
|||
it('does not count a tool that cannot read the type', () => {
|
||||
expect(hasTurnFileConsumer('video/mp4', { executeCode: false, fileSearch: true })).toBe(false);
|
||||
});
|
||||
|
||||
it('counts an enabled tool only where the record shows it holds the file', () => {
|
||||
const consumers = { executeCode: false, fileSearch: true };
|
||||
expect(hasTurnFileConsumer('text/csv', consumers, { embedded: true })).toBe(true);
|
||||
expect(hasTurnFileConsumer('text/csv', consumers, { embedded: false })).toBe(false);
|
||||
expect(hasTurnFileConsumer('text/csv', consumers, {})).toBe(false);
|
||||
});
|
||||
|
||||
it('pairs the evidence with the tool that can read the type', () => {
|
||||
/* Only the sandbox holds this file, so the tool that can read an email export is the one
|
||||
* without a copy of it, while csv is served by the tool that has one. */
|
||||
const held = { metadata: { codeEnvRef: codeRef } };
|
||||
const both = { executeCode: true, fileSearch: true };
|
||||
expect(hasTurnFileConsumer(eml, both, held)).toBe(false);
|
||||
expect(hasTurnFileConsumer('text/csv', both, held)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasToolResourceProvisioning', () => {
|
||||
it('reads vectors for file search and a sandbox pointer for code', () => {
|
||||
expect(hasToolResourceProvisioning({ embedded: true }, EToolResources.file_search)).toBe(true);
|
||||
expect(
|
||||
hasToolResourceProvisioning(
|
||||
{ metadata: { embeddedEntities: ['agent_1'] } },
|
||||
EToolResources.file_search,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
hasToolResourceProvisioning(
|
||||
{ metadata: { codeEnvRef: codeRef } },
|
||||
EToolResources.execute_code,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
hasToolResourceProvisioning(
|
||||
{ metadata: { codeEnvRefs: { default: codeRef } } },
|
||||
EToolResources.execute_code,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not read one tool's store as the other's", () => {
|
||||
expect(hasToolResourceProvisioning({ embedded: true }, EToolResources.execute_code)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
hasToolResourceProvisioning(
|
||||
{ metadata: { codeEnvRef: codeRef } },
|
||||
EToolResources.file_search,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('treats a record with neither as unprovisioned', () => {
|
||||
expect(hasToolResourceProvisioning({}, EToolResources.file_search)).toBe(false);
|
||||
expect(hasToolResourceProvisioning({ embedded: false }, EToolResources.file_search)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
hasToolResourceProvisioning(
|
||||
{ metadata: { embeddedEntities: [] } },
|
||||
EToolResources.file_search,
|
||||
),
|
||||
).toBe(false);
|
||||
expect(hasToolResourceProvisioning({ metadata: {} }, EToolResources.execute_code)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasInferredLLMDeliveryPath', () => {
|
||||
|
|
@ -838,24 +915,71 @@ describe('resolveTurnLLMDeliveryPath', () => {
|
|||
).toBe('none');
|
||||
});
|
||||
|
||||
it('leaves the file to Run Code when the turn can read it with code', () => {
|
||||
it('leaves the file to Run Code when the sandbox it runs on holds the file', () => {
|
||||
expect(
|
||||
resolveTurnLLMDeliveryPath({
|
||||
file: routedCsv,
|
||||
file: { ...routedCsv, metadata: { ...routedCsv.metadata, codeEnvRef: codeRef } },
|
||||
consumers: { executeCode: true, fileSearch: false },
|
||||
endpointConfig,
|
||||
}),
|
||||
).toBe('none');
|
||||
});
|
||||
|
||||
it('leaves the file to File Search when the turn can read it by retrieval', () => {
|
||||
it('leaves the file to File Search once the vector store holds it', () => {
|
||||
expect(
|
||||
resolveTurnLLMDeliveryPath({
|
||||
file: { ...routedCsv, embedded: true },
|
||||
consumers: { executeCode: false, fileSearch: true },
|
||||
endpointConfig,
|
||||
}),
|
||||
).toBe('none');
|
||||
expect(
|
||||
resolveTurnLLMDeliveryPath({
|
||||
file: { ...routedCsv, metadata: { ...routedCsv.metadata, embeddedEntities: ['agent_1'] } },
|
||||
consumers: { executeCode: false, fileSearch: true },
|
||||
endpointConfig,
|
||||
}),
|
||||
).toBe('none');
|
||||
});
|
||||
|
||||
it('delivers text when File Search is on but never received the file', () => {
|
||||
/* The plain-chat File Search toggle: the upload names no destination, so nothing files it
|
||||
* under a tool resource and it is never embedded. Withholding the text on the strength of
|
||||
* the toggle alone left the attachment readable by nothing at all. */
|
||||
expect(
|
||||
resolveTurnLLMDeliveryPath({
|
||||
file: routedCsv,
|
||||
consumers: { executeCode: false, fileSearch: true },
|
||||
endpointConfig,
|
||||
}),
|
||||
).toBe('none');
|
||||
).toBe('text');
|
||||
});
|
||||
|
||||
it('delivers text when Run Code is on but no sandbox holds the file', () => {
|
||||
/* A promoted destination is not uploaded to the sandbox at upload time, so the turn the
|
||||
* file arrives on carries no reference either. */
|
||||
expect(
|
||||
resolveTurnLLMDeliveryPath({
|
||||
file: routedCsv,
|
||||
consumers: { executeCode: true, fileSearch: false },
|
||||
endpointConfig,
|
||||
}),
|
||||
).toBe('text');
|
||||
});
|
||||
|
||||
it('delivers text where the tool holding the file cannot read this type', () => {
|
||||
/* The vectors belong to file search, which this turn does not run, and code execution both
|
||||
* lacks a copy and cannot read an email export, so nothing here serves the file. */
|
||||
expect(
|
||||
resolveTurnLLMDeliveryPath({
|
||||
file: { ...routedCsv, type: eml, embedded: true },
|
||||
consumers: { executeCode: true, fileSearch: false },
|
||||
endpointConfig: {
|
||||
...endpointConfig,
|
||||
defaultLLMDeliveryPath: { overrides: { [eml]: 'none' } },
|
||||
},
|
||||
}),
|
||||
).toBe('text');
|
||||
});
|
||||
|
||||
it('does not judge a turn whose tools are unknown', () => {
|
||||
|
|
@ -934,6 +1058,7 @@ describe('resolveTurnLLMDeliveryPath', () => {
|
|||
const converted = {
|
||||
...routedCsv,
|
||||
type: 'image/png',
|
||||
embedded: true,
|
||||
metadata: { destinationChosen: false, routingMimeType: 'text/csv' },
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config';
|
||||
import type { EndpointFileConfig, FileConfig, RegexLike } from './types/files';
|
||||
import type { CodeEnvReferenceSet } from './codeEnvRef';
|
||||
import type { TEndpoint } from './config';
|
||||
import {
|
||||
retrievalMimeTypes,
|
||||
|
|
@ -17,6 +18,7 @@ import {
|
|||
} from './schemas';
|
||||
import { normalizeEndpointName } from './utils';
|
||||
import { EToolResources } from './types/tools';
|
||||
import { getCodeEnvRefs } from './codeEnvRef';
|
||||
|
||||
/**
|
||||
* The native provider a custom endpoint declares, when it declares one. A custom endpoint
|
||||
|
|
@ -362,11 +364,42 @@ export interface TurnFileConsumers {
|
|||
fileSearch: boolean;
|
||||
}
|
||||
|
||||
/** Whether a tool this turn runs can read a file of this type. */
|
||||
export function hasTurnFileConsumer(mimeType: string, consumers: TurnFileConsumers): boolean {
|
||||
/**
|
||||
* Whether the record shows this file reached a tool's own store: vectors for file search, a
|
||||
* sandbox pointer for code execution. Nothing else writes either, so their presence is proof
|
||||
* the file was provisioned and their absence proof it was not. The same evidence deferred
|
||||
* provisioning reads before queueing a file, so a turn cannot withhold content for a tool that
|
||||
* provisioning has yet to serve.
|
||||
*/
|
||||
export function hasToolResourceProvisioning(file: TurnDeliveryFile, toolResource: string): boolean {
|
||||
if (toolResource === EToolResources.execute_code) {
|
||||
return getCodeEnvRefs(file.metadata).length > 0;
|
||||
}
|
||||
return file.embedded === true || (file.metadata?.embeddedEntities?.length ?? 0) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a tool this turn runs can read a file of this type.
|
||||
*
|
||||
* Passing the record asks the stricter question a delivery decision needs: a tool serves a file
|
||||
* only once it holds it. A file tool being enabled is not the same as the file having reached
|
||||
* it, and an upload that named no destination is filed under no tool at all, so judging the
|
||||
* tool set alone reads an empty vector store as a reader.
|
||||
*/
|
||||
export function hasTurnFileConsumer(
|
||||
mimeType: string,
|
||||
consumers: TurnFileConsumers,
|
||||
file?: TurnDeliveryFile,
|
||||
): boolean {
|
||||
const holds = (toolResource: EToolResources): boolean =>
|
||||
file == null || hasToolResourceProvisioning(file, toolResource);
|
||||
return (
|
||||
(consumers.executeCode && canToolResourceConsume(EToolResources.execute_code, mimeType)) ||
|
||||
(consumers.fileSearch && canToolResourceConsume(EToolResources.file_search, mimeType))
|
||||
(consumers.executeCode &&
|
||||
canToolResourceConsume(EToolResources.execute_code, mimeType) &&
|
||||
holds(EToolResources.execute_code)) ||
|
||||
(consumers.fileSearch &&
|
||||
canToolResourceConsume(EToolResources.file_search, mimeType) &&
|
||||
holds(EToolResources.file_search))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -394,7 +427,17 @@ export interface TurnDeliveryFile {
|
|||
text?: string | null;
|
||||
/** Stored as an upload-time inference, so any string may be read back. */
|
||||
llmDeliveryPath?: string | null;
|
||||
metadata?: { routingMimeType?: string; destinationChosen?: boolean } | null;
|
||||
/** Whether a vector store holds this file, which is what lets file search serve it.
|
||||
* Records predating namespace tracking carry only this flag. */
|
||||
embedded?: boolean | null;
|
||||
metadata?:
|
||||
| ({
|
||||
routingMimeType?: string;
|
||||
destinationChosen?: boolean;
|
||||
/** Vector namespaces holding this file, written as each embedding succeeds. */
|
||||
embeddedEntities?: string[];
|
||||
} & CodeEnvReferenceSet)
|
||||
| null;
|
||||
}
|
||||
|
||||
const isLLMDeliveryPath = (value: unknown): value is TDefaultLLMDeliveryPath =>
|
||||
|
|
@ -410,10 +453,16 @@ export function hasInferredLLMDeliveryPath(file: TurnDeliveryFile): boolean {
|
|||
*
|
||||
* A record predating routing and a destination the user chose keep what they stored. An
|
||||
* inferred route re-resolves against the endpoint handling the turn. A `none` route leaves
|
||||
* the file for a tool; where the endpoint enables `textFallbackWithoutTools` and this turn
|
||||
* runs no tool that can read the file, the text extracted at upload is delivered rather than
|
||||
* the file reaching nothing. Consumers left undefined are unknown and not judged, as in
|
||||
* {@link resolveUploadDestination}.
|
||||
* the file for a tool; where the endpoint enables `textFallbackWithoutTools` and no tool this
|
||||
* turn runs both reads the file and already holds it, the text extracted at upload is delivered
|
||||
* rather than the file reaching nothing. Consumers left undefined are unknown and not judged, as
|
||||
* in {@link resolveUploadDestination}.
|
||||
*
|
||||
* Holding the file is asked of the record rather than of the tool set, because the two disagree
|
||||
* for the upload that needs the fallback most: one that named no destination is filed under no
|
||||
* tool, so enabling file search would otherwise withhold the text for a vector store that never
|
||||
* received the file, leaving the attachment readable by nothing. Deferred provisioning still
|
||||
* fills that store on first use, which delivering the text does not prevent.
|
||||
*/
|
||||
export function resolveTurnLLMDeliveryPath(
|
||||
routing: Partial<TurnDeliveryRouting> | undefined,
|
||||
|
|
@ -433,7 +482,7 @@ export function resolveTurnLLMDeliveryPath(
|
|||
endpointConfig?.textFallbackWithoutTools === true &&
|
||||
consumers != null &&
|
||||
hasFallbackText &&
|
||||
!hasTurnFileConsumer(mimeType, consumers)
|
||||
!hasTurnFileConsumer(mimeType, consumers, file)
|
||||
) {
|
||||
return 'text';
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue