diff --git a/api/package.json b/api/package.json index 95aa456d2b..5b545f9b28 100644 --- a/api/package.json +++ b/api/package.json @@ -40,7 +40,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.2.1", + "@librechat/agents": "^4.2.2", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index 20b5c73f68..e791be8047 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -196,6 +196,7 @@ class ModelEndHandler { ? { cache_creation, cache_creation_1h: taggedUsage.input_token_details?.cache_creation_1h, + cache_creation_12h: taggedUsage.input_token_details?.cache_creation_12h, cache_read, } : undefined, diff --git a/package-lock.json b/package-lock.json index 656c0a7c2e..e6abf57203 100644 --- a/package-lock.json +++ b/package-lock.json @@ -70,7 +70,7 @@ "@azure/storage-blob": "^12.30.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.2.1", + "@librechat/agents": "^4.2.2", "@librechat/api": "*", "@librechat/data-schemas": "*", "@microsoft/microsoft-graph-client": "^3.0.7", @@ -9479,9 +9479,9 @@ } }, "node_modules/@librechat/agents": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-4.2.1.tgz", - "integrity": "sha512-HDgD95i7Vx9cx/SqgV1LlQG4fS9ZixJANxFP6wfS8dBXJCeqY27j/lYwtmyswTo45Q7dDRTR6XnN/qYsMToAHg==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-4.2.2.tgz", + "integrity": "sha512-Yx248RpKQo0UAGNpPny5Y5anmYVXTbg5UhGJXeeYdxX7wKn+cBvM691jCWPTBWgps5D0Q4p5gj0T4Y2kyUTksw==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.115.0", @@ -42372,7 +42372,7 @@ "@azure/identity": "^4.14.0", "@azure/storage-blob": "^12.30.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.2.1", + "@librechat/agents": "^4.2.2", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.32.1", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/package.json b/packages/api/package.json index 26dd792e0f..64d19b0467 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -113,7 +113,7 @@ "@azure/identity": "^4.14.0", "@azure/storage-blob": "^12.30.0", "@keyv/redis": "5.1.6", - "@librechat/agents": "^4.2.1", + "@librechat/agents": "^4.2.2", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.32.1", "@opentelemetry/api": "^1.9.0", diff --git a/packages/api/src/agents/__tests__/run-summarization.test.ts b/packages/api/src/agents/__tests__/run-summarization.test.ts index 33b1922a34..90e96ee850 100644 --- a/packages/api/src/agents/__tests__/run-summarization.test.ts +++ b/packages/api/src/agents/__tests__/run-summarization.test.ts @@ -5,6 +5,7 @@ import { CallbackManager } from '@langchain/core/callbacks/manager'; import { encryptV3, logger, mergeConfigOverrides } from '@librechat/data-schemas'; import { EModelEndpoint, + ReasoningEffort, FileSources, PrincipalType, MAX_SUBAGENT_DEPTH, @@ -1078,6 +1079,648 @@ async function compactSummary( return { requests }; } +describe('final Cyber summary configuration', () => { + it.each([ + { sourceModel: 'claude-sonnet-4.6', sourceProvider: Providers.ANTHROPIC, carrier: 'baseURL' }, + { + sourceModel: 'claude-sonnet-4.6', + sourceProvider: Providers.ANTHROPIC, + carrier: 'configuration', + }, + { sourceModel: 'claude-sonnet-4.6', sourceProvider: Providers.ANTHROPIC, carrier: 'default' }, + { sourceModel: 'gpt-6-cyber', sourceProvider: Providers.OPENAI, carrier: 'baseURL' }, + { sourceModel: 'gpt-6-cyber', sourceProvider: Providers.OPENAI, carrier: 'configuration' }, + { sourceModel: 'gpt-6-cyber', sourceProvider: Providers.OPENAI, carrier: 'default' }, + ])( + 'enforces native identity after final parameter merge: %j', + async ({ sourceModel, sourceProvider, carrier }) => { + const nativeSource = sourceProvider === Providers.OPENAI; + const identity = { userId: 'final-summary-user', tenantId: 'request-tenant' }; + const { llmConfig, configOptions } = getOpenAIConfig( + 'resolved-user-key', + { + requestIdentity: identity, + modelOptions: { model: sourceModel }, + headers: { 'X-Final-Transport': 'inherited' }, + }, + EModelEndpoint.openAI, + ); + const mainOptions = nativeSource + ? { ...llmConfig, configuration: configOptions } + : { + model: sourceModel, + apiKey: 'anthropic-key', + modelKwargs: { source_only: 'do not inherit' }, + }; + const snapshot = { ...mainOptions, modelKwargs: { ...mainOptions.modelKwargs } }; + const summaryParameters: TestSummarizationParameters = { + streaming: false, + apiKey: 'summary-key', + user: 'spoofed-user', + safety_identifier: 'shared-literal', + ...(carrier === 'baseURL' && { baseURL: 'https://api.openai.com/v1' }), + ...(carrier === 'configuration' && { + configuration: { + baseURL: 'https://api.openai.com/v1', + defaultHeaders: { 'X-Final-Transport': 'override' }, + defaultQuery: { scope: 'summary' }, + }, + }), + }; + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: sourceProvider, + endpoint: nativeSource ? EModelEndpoint.openAI : EModelEndpoint.anthropic, + model: sourceModel, + model_parameters: mainOptions, + }), + ], + user: { id: identity.userId, tenantId: 'stale-user-tenant' } as IUser, + tenantId: identity.tenantId, + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + provider: EModelEndpoint.openAI, + model: 'gpt-6-cyber', + parameters: summaryParameters as SummarizationConfig['parameters'], + }, + }); + const { requests } = await compactSummary(agents); + expect(requests).toHaveLength(1); + expect(requests[0].url.origin + requests[0].url.pathname).toBe( + 'https://api.openai.com/v1/responses', + ); + expect(requests[0].body).toMatchObject({ + model: 'gpt-6-cyber', + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: + llmConfig.modelKwargs?.safety_identifier ?? + getOpenAIConfig( + '', + { requestIdentity: identity, modelOptions: { model: 'gpt-6-cyber' } }, + EModelEndpoint.openAI, + ).llmConfig.modelKwargs?.safety_identifier, + }); + expect(requests[0].body).not.toHaveProperty('user'); + expect(requests[0].body).not.toHaveProperty('source_only'); + if (carrier === 'configuration') { + expect(requests[0].headers.get('X-Final-Transport')).toBe('override'); + expect(requests[0].url.searchParams.get('scope')).toBe('summary'); + } else if (nativeSource) { + expect(requests[0].headers.get('X-Final-Transport')).toBe('inherited'); + } + expect(mainOptions).toEqual(snapshot); + expect(summaryParameters.safety_identifier).toBe('shared-literal'); + }, + ); +}); + +describe('Cyber summary final transport boundaries', () => { + it.each([false, true])( + 'preserves self-summary API selection while replacing shared identity (%s)', + async (useResponsesApi) => { + const identity = { userId: 'self-summary-user', tenantId: 'self-summary-tenant' }; + const { llmConfig, configOptions } = getOpenAIConfig( + 'resolved-user-key', + { + requestIdentity: identity, + modelOptions: { model: 'gpt-6-cyber' }, + addParams: { useResponsesApi, access_programs: { cyber: 'standard' } }, + }, + EModelEndpoint.openAI, + ); + const summaryParameters: TestSummarizationParameters = { + streaming: false, + modelKwargs: { + model: 'gpt-6-sol', + safety_identifier: 'nested-shared-literal', + access_programs: { cyber: 'standard' }, + }, + safety_identifier: 'scalar-shared-literal', + }; + const before = { ...llmConfig.modelKwargs }; + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: Providers.OPENAI, + endpoint: EModelEndpoint.openAI, + model: 'gpt-6-cyber', + model_parameters: { ...llmConfig, configuration: configOptions }, + }), + ], + user: { id: identity.userId, tenantId: identity.tenantId } as IUser, + tenantId: identity.tenantId, + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + model: 'gpt-6-cyber', + parameters: summaryParameters as SummarizationConfig['parameters'], + }, + }); + const { requests } = await compactSummary(agents); + expect(requests[0].url.pathname).toBe( + useResponsesApi ? '/v1/responses' : '/v1/chat/completions', + ); + expect(requests[0].body).toMatchObject({ + model: 'gpt-6-cyber', + access_programs: { cyber: 'standard' }, + safety_identifier: llmConfig.modelKwargs?.safety_identifier, + }); + expect(llmConfig.modelKwargs).toEqual(before); + expect(summaryParameters.safety_identifier).toBe('scalar-shared-literal'); + }, + ); + + it.each(['baseURL', 'configuration', 'sdk-default'])( + 'does not apply the native contract to a cross-provider gateway (%s)', + async (carrier) => { + if (carrier === 'sdk-default') + jest.replaceProperty(process, 'env', { + ...process.env, + OPENAI_BASE_URL: 'https://summary-gateway.example/v1', + }); + const summaryParameters: TestSummarizationParameters = { + streaming: false, + apiKey: 'gateway-key', + safety_identifier: 'gateway-owned-identity', + ...(carrier === 'baseURL' && { baseURL: 'https://summary-gateway.example/v1' }), + ...(carrier === 'configuration' && { + configuration: { baseURL: 'https://summary-gateway.example/v1' }, + }), + }; + try { + const agents = await callAndCapture({ + agents: [ + makeAgent({ + provider: Providers.ANTHROPIC, + endpoint: EModelEndpoint.anthropic, + model: 'claude-sonnet-4.6', + model_parameters: { model: 'claude-sonnet-4.6' }, + }), + ], + user: { id: 'gateway-summary-user', tenantId: 'gateway-summary-tenant' } as IUser, + tenantId: 'gateway-summary-tenant', + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + provider: EModelEndpoint.openAI, + model: 'gpt-6-cyber', + parameters: summaryParameters as SummarizationConfig['parameters'], + }, + }); + const { requests } = await compactSummary(agents); + expect(requests[0].url.origin).toBe('https://summary-gateway.example'); + expect(requests[0].body).not.toHaveProperty('access_programs'); + expect(requests[0].body.safety_identifier).toBe('gateway-owned-identity'); + } finally { + if (carrier === 'sdk-default') jest.restoreAllMocks(); + } + }, + ); +}); + +describe('Cyber summary SDK URL defaults', () => { + beforeEach(() => { + const env: NodeJS.ProcessEnv = { ...process.env, OPENAI_API_KEY: 'summary-key' }; + for (const key of [ + 'OPENAI_API_BASE', + 'OPENAI_BASE_URL', + 'OPENAI_REVERSE_PROXY', + 'LANGSMITH_GATEWAY', + ]) { + delete env[key]; + } + jest.replaceProperty(process, 'env', env); + }); + afterEach(() => jest.restoreAllMocks()); + + it.each([ + { + apiBase: 'https://summary-api-base.example/v1', + sdkURL: 'https://api.openai.com/v1', + explicitURL: undefined, + gateway: undefined, + origin: 'https://summary-api-base.example', + native: false, + }, + { + apiBase: '', + sdkURL: 'https://summary-sdk-url.example/v1', + explicitURL: undefined, + gateway: undefined, + origin: 'https://summary-sdk-url.example', + native: false, + }, + { + apiBase: 'https://api.openai.com/v1', + sdkURL: 'https://summary-sdk-url.example/v1', + explicitURL: undefined, + gateway: undefined, + origin: 'https://api.openai.com', + native: true, + }, + { + apiBase: 'https://summary-api-base.example/v1', + sdkURL: undefined, + explicitURL: 'https://api.openai.com/v1', + gateway: undefined, + origin: 'https://api.openai.com', + native: true, + }, + { + apiBase: 'https://api.openai.com/v1', + sdkURL: undefined, + explicitURL: 'https://summary-explicit.example/v1', + gateway: undefined, + origin: 'https://summary-explicit.example', + native: false, + }, + { + apiBase: undefined, + sdkURL: undefined, + explicitURL: undefined, + gateway: 'true', + origin: 'https://gateway.smith.langchain.com', + native: false, + }, + { + apiBase: undefined, + sdkURL: undefined, + explicitURL: undefined, + gateway: 'false', + origin: 'https://api.openai.com', + native: true, + }, + ])( + 'shapes cross-provider summaries for the actual SDK destination: %j', + async ({ apiBase, sdkURL, explicitURL, gateway, origin, native }) => { + for (const [key, value] of Object.entries({ + OPENAI_API_BASE: apiBase, + OPENAI_BASE_URL: sdkURL, + LANGSMITH_GATEWAY: gateway, + })) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + const parameters: TestSummarizationParameters = { + streaming: false, + apiKey: 'summary-key', + ...(explicitURL && { configuration: { baseURL: explicitURL } }), + }; + const agents = await callAndCapture({ + agents: [ + makeAgent({ + provider: Providers.ANTHROPIC, + endpoint: EModelEndpoint.anthropic, + model: 'claude-sonnet-4.6', + model_parameters: { model: 'claude-sonnet-4.6' }, + }), + ], + user: { id: 'summary-url-user', tenantId: 'summary-url-tenant' } as IUser, + tenantId: 'summary-url-tenant', + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + provider: EModelEndpoint.openAI, + model: 'gpt-6-cyber', + parameters: parameters as SummarizationConfig['parameters'], + }, + }); + const { requests } = await compactSummary(agents); + expect(requests).toHaveLength(1); + expect(requests[0].url.origin).toBe(origin); + expect(requests[0].url.pathname.endsWith(native ? '/responses' : '/chat/completions')).toBe( + true, + ); + if (native) { + expect(requests[0].body).toMatchObject({ + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + } else { + expect(requests[0].body).not.toHaveProperty('access_programs'); + expect(requests[0].body).not.toHaveProperty('safety_identifier'); + } + }, + ); +}); + +describe.each([false, true])( + 'Cyber same-endpoint routing policy (explicit provider=%s)', + (explicitProvider) => { + it.each([ + { sourceModel: 'gpt-6-sol', policy: false, summary: undefined, expected: false }, + { sourceModel: 'gpt-4.1-mini', policy: false, summary: undefined, expected: false }, + { sourceModel: 'gpt-6-cyber', policy: false, summary: undefined, expected: false }, + { sourceModel: 'gpt-6-sol', policy: true, summary: undefined, expected: true }, + { sourceModel: 'gpt-6-sol', policy: false, summary: true, expected: true }, + { sourceModel: 'gpt-6-sol', policy: true, summary: false, expected: false }, + { sourceModel: 'gpt-4.1-mini', policy: undefined, summary: undefined, expected: true }, + { sourceModel: 'gpt-6-sol', policy: 'default', summary: undefined, expected: false }, + { sourceModel: 'gpt-6-sol', policy: 'drop', summary: undefined, expected: false }, + { sourceModel: 'gpt-6-sol', policy: 'drop', summary: true, expected: true }, + ])( + 'preserves the target endpoint policy without changing the agent: %j', + async ({ sourceModel, policy, summary, expected }) => { + const endpoint: TestCustomEndpoint = { + name: 'Daybreak', + apiKey: 'endpoint-key', + baseURL: 'https://api.openai.com/v1', + addParams: { + ...(typeof policy === 'boolean' && { useResponsesApi: policy }), + ...(policy === 'drop' && { useResponsesApi: true }), + }, + dropParams: policy === 'drop' ? ['useResponsesApi'] : undefined, + customParams: + policy === 'default' + ? { + defaultParamsEndpoint: 'custom', + paramDefinitions: [{ key: 'useResponsesApi', default: false }], + } + : undefined, + }; + const endpointParams = { ...endpoint.addParams }; + const user = { id: 'route-user', tenantId: 'route-tenant' } as IUser; + const { llmConfig, configOptions } = getOpenAIConfig( + endpoint.apiKey, + { + requestIdentity: { userId: user.id, tenantId: user.tenantId }, + modelOptions: { model: sourceModel }, + reverseProxyUrl: endpoint.baseURL, + addParams: endpoint.addParams, + dropParams: endpoint.dropParams, + customParams: endpoint.customParams, + headers: { 'X-Route-Transport': 'preserved' }, + }, + endpoint.name, + ); + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: Providers.OPENAI, + endpoint: endpoint.name, + model: sourceModel, + model_parameters: { ...llmConfig, configuration: configOptions }, + }), + ], + user, + tenantId: user.tenantId, + appConfig: makeAppConfig([endpoint]), + summarizeOnly: true, + summarizationConfig: { + ...(explicitProvider && { provider: endpoint.name }), + model: 'gpt-6-cyber', + parameters: { + streaming: false, + ...(summary !== undefined && { useResponsesApi: summary }), + }, + }, + }); + const sourceApiMode = llmConfig.useResponsesApi; + const { requests } = await compactSummary(agents); + expect(requests).toHaveLength(1); + expect(requests[0].url.origin + requests[0].url.pathname).toBe( + `https://api.openai.com/v1/${expected ? 'responses' : 'chat/completions'}`, + ); + expect(requests[0].headers.get('X-Route-Transport')).toBe('preserved'); + expect(requests[0].body).toMatchObject({ + model: 'gpt-6-cyber', + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect((agents[0].clientOptions as Record).useResponsesApi).toBe( + sourceApiMode, + ); + expect(endpoint.addParams).toEqual(endpointParams); + }, + ); + }, +); + +describe('Cyber summary target shaping', () => { + it.each([ + { sourceModel: 'gpt-6-sol', baseURL: undefined, endpoint: EModelEndpoint.openAI }, + { sourceModel: 'gpt-4.1-mini', baseURL: undefined, endpoint: EModelEndpoint.openAI }, + { sourceModel: 'gpt-6-sol', baseURL: 'https://api.openai.com/v1', endpoint: 'Daybreak' }, + ])( + 'shapes a Cyber summary while preserving $sourceModel transport', + async ({ sourceModel, baseURL, endpoint }) => { + const user = { id: 'summary-user', tenantId: 'summary-tenant' } as IUser; + const { llmConfig, configOptions } = getOpenAIConfig( + 'resolved-user-key', + { + modelOptions: { model: sourceModel, reasoning_effort: ReasoningEffort.high }, + reverseProxyUrl: baseURL, + headers: { 'X-Summary-Transport': 'kept' }, + defaultQuery: { 'request-scope': 'kept' }, + addParams: { prompt_cache_options: { ttl: '12h' } }, + }, + endpoint, + ); + const before = { ...llmConfig.modelKwargs }; + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: Providers.OPENAI, + endpoint, + model: sourceModel, + model_parameters: { ...llmConfig, configuration: configOptions }, + }), + ], + user, + tenantId: user.tenantId, + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + provider: endpoint, + model: 'gpt-6-cyber', + parameters: { streaming: false }, + }, + }); + const { requests } = await compactSummary(agents); + expect(requests).toHaveLength(1); + expect(requests[0].url.pathname).toBe('/v1/responses'); + expect(requests[0].url.searchParams.get('request-scope')).toBe('kept'); + expect(requests[0].headers.get('X-Summary-Transport')).toBe('kept'); + expect(requests[0].body).toMatchObject({ + model: 'gpt-6-cyber', + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: expect.stringMatching(/^[a-f0-9]{64}$/), + prompt_cache_options: { ttl: '12h' }, + }); + expect(requests[0].body).not.toHaveProperty('reasoning'); + expect(llmConfig.modelKwargs).toEqual(before); + expect((agents[0].clientOptions as Record).modelKwargs).toEqual(before); + }, + ); + + it('preserves an explicit summary API selection and server-owned identity', async () => { + const { llmConfig, configOptions } = getOpenAIConfig( + 'resolved-user-key', + { + modelOptions: { model: 'gpt-6-sol' }, + headers: { 'X-Summary-Transport': 'kept' }, + }, + EModelEndpoint.openAI, + ); + const summaryKwargs = { + access_programs: { cyber: 'standard' }, + safety_identifier: 'shared-spoof', + }; + const summaryParameters: TestSummarizationParameters = { + streaming: false, + useResponsesApi: false, + modelKwargs: summaryKwargs, + }; + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: Providers.OPENAI, + endpoint: EModelEndpoint.openAI, + model: 'gpt-6-sol', + model_parameters: { ...llmConfig, configuration: configOptions }, + }), + ], + user: { id: 'summary-user', tenantId: 'stale-user-tenant' } as IUser, + tenantId: 'request-tenant', + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + model: 'gpt-6-cyber', + parameters: summaryParameters as SummarizationConfig['parameters'], + }, + }); + const expected = getOpenAIConfig( + 'unused', + { + requestIdentity: { userId: 'summary-user', tenantId: 'request-tenant' }, + modelOptions: { model: 'gpt-6-cyber' }, + }, + EModelEndpoint.openAI, + ); + const { requests } = await compactSummary(agents); + expect(requests[0].url.pathname).toBe('/v1/chat/completions'); + expect(requests[0].headers.get('X-Summary-Transport')).toBe('kept'); + expect(requests[0].body).toMatchObject({ + access_programs: { cyber: 'standard' }, + safety_identifier: expected.llmConfig.modelKwargs?.safety_identifier, + }); + expect(summaryKwargs).toEqual({ + access_programs: { cyber: 'standard' }, + safety_identifier: 'shared-spoof', + }); + }); + + it('rejects a native Cyber summary without authenticated identity before calling the SDK', async () => { + await expect( + callAndCapture({ + agents: [makeAgent({ model: 'gpt-6-sol', model_parameters: { model: 'gpt-6-sol' } })], + appConfig: makeAppConfig([]), + summarizationConfig: { model: 'gpt-6-cyber' }, + }), + ).rejects.toThrow('GPT-6 Cyber requires an authenticated user.'); + expect(Run.create).not.toHaveBeenCalled(); + }); + + it('leaves a disabled Cyber summary inert without an authenticated identity', async () => { + const agents = await callAndCapture({ + agents: [makeAgent({ model: 'gpt-6-sol', model_parameters: { model: 'gpt-6-sol' } })], + appConfig: makeAppConfig([]), + summarizationConfig: { enabled: false, model: 'gpt-6-cyber' }, + }); + expect(agents[0].summarizationEnabled).toBe(false); + }); + + it.each(['Gateway', EModelEndpoint.openAI])( + 'keeps gateway transport outside native Cyber shaping (%s)', + async (endpoint) => { + const { llmConfig, configOptions } = getOpenAIConfig( + 'gateway-key', + { + modelOptions: { model: 'gpt-4.1-mini' }, + reverseProxyUrl: endpoint === 'Gateway' ? 'https://gateway.example/v1' : undefined, + }, + endpoint, + ); + const summaryParameters: TestSummarizationParameters = { + streaming: false, + ...(endpoint === EModelEndpoint.openAI && { + configuration: { baseURL: 'https://gateway.example/v1' }, + }), + }; + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: Providers.OPENAI, + endpoint, + model: 'gpt-4.1-mini', + model_parameters: { ...llmConfig, configuration: configOptions }, + }), + ], + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { + provider: endpoint, + model: 'gpt-6-cyber', + parameters: summaryParameters as SummarizationConfig['parameters'], + }, + }); + const { requests } = await compactSummary(agents); + expect(requests[0].url.origin).toBe('https://gateway.example'); + expect(requests[0].url.pathname).toBe('/v1/chat/completions'); + expect(requests[0].body).not.toHaveProperty('access_programs'); + expect(requests[0].body).not.toHaveProperty('safety_identifier'); + }, + ); +}); + +describe('Cyber summary model isolation', () => { + it.each(['gpt-6-sol', 'gpt-4.1-mini', 'gpt-6-cyber'])( + 'scopes Cyber access to the matching summary model, %s', + async (model) => { + const { llmConfig, configOptions } = getOpenAIConfig( + 'test-cyber-key', + { + requestIdentity: { userId: 'test-user', tenantId: 'test-tenant' }, + modelOptions: { model: 'gpt-6-cyber' }, + headers: { 'X-Test-Transport': 'preserved' }, + }, + EModelEndpoint.openAI, + ); + const agents = await callAndCapture({ + agents: [ + makeReasoningAgent({ + provider: Providers.OPENAI, + endpoint: EModelEndpoint.openAI, + model: 'gpt-6-cyber', + model_parameters: { ...llmConfig, configuration: configOptions }, + }), + ], + user: { id: 'test-user', tenantId: 'test-tenant' } as IUser, + tenantId: 'test-tenant', + appConfig: makeAppConfig([]), + summarizeOnly: true, + summarizationConfig: { model, parameters: { streaming: false } }, + }); + + const { requests } = await compactSummary(agents); + expect(requests).toHaveLength(1); + expect(requests[0].body.model).toBe(model); + expect(requests[0].url.origin).toBe('https://api.openai.com'); + expect(requests[0].headers.get('X-Test-Transport')).toBe('preserved'); + if (model === 'gpt-6-cyber') { + expect(requests[0].body).toHaveProperty('access_programs.cyber', 'daybreak_red'); + } else { + expect(requests[0].body).not.toHaveProperty('access_programs'); + } + expect((agents[0].clientOptions as Record).modelKwargs).toMatchObject({ + access_programs: { cyber: 'daybreak_red' }, + }); + }, + ); +}); + describe('manual compaction recency', () => { it('skips a fully retained conversation without calling the summarizer', async () => { const agents = await callAndCapture({ diff --git a/packages/api/src/agents/cache.spec.ts b/packages/api/src/agents/cache.spec.ts new file mode 100644 index 0000000000..2124be0cc9 --- /dev/null +++ b/packages/api/src/agents/cache.spec.ts @@ -0,0 +1,205 @@ +import { Providers, initializeModel } from '@librechat/agents'; +import { AIMessage } from '@librechat/agents/langchain/messages'; +import type { UsageMetadata } from '~/stream/interfaces/IJobStore'; +import type { OpenAIConfiguration } from '~/types'; +import { getModelEndUsage, recordCollectedUsage, computeUsageCostUSD } from './usage'; +import { createTxMethods } from '../../../data-schemas/src/methods/tx'; +import { matchModelName, findMatchingPattern } from '~/utils/tokens'; + +const model = 'gpt-6-cyber'; +const pricing = createTxMethods({} as typeof import('mongoose'), { + matchModelName: (name) => matchModelName(name), + findMatchingPattern: (name, values) => findMatchingPattern(name, values) ?? undefined, +}); + +async function getSDKUsage( + streaming: boolean, + input: number, + write: number, + twelveHour: number, + read = 0, +) { + const response = { + id: 'resp_cache', + object: 'response', + status: 'completed', + model, + output: [], + usage: { + input_tokens: input, + output_tokens: 0, + total_tokens: input, + input_tokens_details: { + cached_tokens: read, + cache_write_tokens: write, + cache_write_tokens_details: { + ttl_30m_tokens: write - twelveHour, + ttl_12h_tokens: twelveHour, + }, + }, + }, + }; + const fetch: NonNullable['fetch']> = async (_url, init) => { + const body = JSON.parse(String(init?.body)); + expect(body.prompt_cache_options.ttl).toBe('12h'); + expect(body.stream).toBe(streaming); + if (!streaming) return Response.json(response); + return new Response( + [ + { type: 'response.created', response: { ...response, status: 'in_progress' } }, + { type: 'response.completed', response }, + ] + .map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + .join(''), + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + }; + const llm = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { + model, + apiKey: 'test-key', + streaming, + useResponsesApi: true, + modelKwargs: { prompt_cache_options: { ttl: '12h' } }, + configuration: { fetch }, + }, + }); + const collected: UsageMetadata[] = []; + const output = await llm.invoke('Verify the patch.', { + callbacks: [ + { + handleLLMEnd(result) { + for (const generations of result.generations) { + for (const generation of generations) { + if ('message' in generation && generation.message instanceof AIMessage) { + const usage = getModelEndUsage( + generation.message as Parameters[0], + ); + if (usage) collected.push(usage); + } + } + } + }, + }, + ], + }); + expect(collected).toHaveLength(1); + const usage = collected[0]; + expect(usage).toEqual(getModelEndUsage(output)); + expect(usage?.input_token_details?.cache_creation).toBe(write); + expect(output.usage_metadata?.input_token_details).toMatchObject({ + cache_creation: write, + cache_creation_12h: twelveHour, + }); + expect(usage.input_token_details?.cache_creation_12h).toBe(twelveHour); + return { ...usage, model, provider: Providers.OPENAI }; +} + +describe.each([false, true])('SDK cache accounting (streaming=%s)', (streaming) => { + it.each([ + { input: 10000, write: 10000, twelveHour: 10000, read: 0, debit: 500000 }, + { input: 12000, write: 7000, twelveHour: 4000, read: 4000, debit: 303000 }, + { input: 272000, write: 7000, twelveHour: 4000, read: 4000, debit: 5503000 }, + { input: 272001, write: 7000, twelveHour: 4000, read: 4000, debit: 10736040 }, + { input: 10000, write: 10000, twelveHour: 0, read: 0, debit: 250000 }, + ])( + 'bills SDK-reported 12-hour writes once: %j', + async ({ input, write, twelveHour, read, debit }) => { + const usage = await getSDKUsage(streaming, input, write, twelveHour, read); + const spendStructuredTokens = jest.fn().mockResolvedValue(undefined); + const insertMany = jest.fn().mockResolvedValue(undefined); + const updateBalance = jest.fn().mockResolvedValue(undefined); + const deps = { spendTokens: jest.fn(), spendStructuredTokens }; + const params = { + user: 'cache-user', + conversationId: 'cache-conversation', + model, + balance: { enabled: true }, + collectedUsage: [usage], + }; + + await recordCollectedUsage(deps, params); + expect(spendStructuredTokens).toHaveBeenCalledWith(expect.objectContaining({ model }), { + promptTokens: { + input: input - write - read, + write, + read, + ...(twelveHour > 0 && { write12h: twelveHour }), + }, + completionTokens: 0, + }); + await recordCollectedUsage( + { ...deps, pricing, bulkWriteOps: { insertMany, updateBalance } }, + params, + ); + expect(insertMany).toHaveBeenCalledTimes(1); + expect(insertMany).toHaveBeenCalledWith([ + expect.objectContaining({ rawAmount: -input, writeTokens: -write, tokenValue: -debit }), + ]); + expect(updateBalance).toHaveBeenCalledTimes(1); + expect(updateBalance).toHaveBeenCalledWith( + expect.objectContaining({ user: 'cache-user', incrementValue: -debit }), + ); + expect(computeUsageCostUSD(usage, pricing)).toBe(debit / 1e6); + if (input > 272000) { + expect( + pricing.getCacheMultiplier({ model, cacheType: 'write12h', inputTokenCount: input }), + ).toBeNull(); + expect(insertMany.mock.calls[0][0][0].rateDetail.write12h).toBe(40); + } + }, + ); +}); + +it.each([-1, 0, 20000])('bounds the 12-hour subset to aggregate writes: %i', (twelveHour) => { + const usage = getModelEndUsage({ + usage_metadata: { input_tokens: 10000, input_token_details: { cache_creation: 10000 } }, + response_metadata: { + usage: { + input_tokens_details: { cache_write_tokens_details: { ttl_12h_tokens: twelveHour } }, + }, + }, + }); + expect(computeUsageCostUSD({ ...usage, model, provider: Providers.OPENAI }, pricing)).toBe( + twelveHour > 0 ? 0.5 : 0.25, + ); +}); + +it('keeps Anthropic 5-minute and 1-hour writes as disjoint shares of aggregate writes', () => { + const usage = getModelEndUsage({ + usage_metadata: { input_tokens: 3000, input_token_details: { cache_creation: 3000 } }, + response_metadata: { + usage: { + cache_creation: { ephemeral_5m_input_tokens: 1000, ephemeral_1h_input_tokens: 2000 }, + }, + }, + }); + expect( + computeUsageCostUSD( + { ...usage, model: 'claude-sonnet-4-5', provider: Providers.ANTHROPIC }, + pricing, + ), + ).toBeCloseTo((1000 * 3.75 + 2000 * 6) / 1e6); +}); + +it('preserves legacy OpenAI usage without a cache-write TTL breakdown', () => { + const usage_metadata: UsageMetadata = { + input_tokens: 1000, + provider: Providers.OPENAI, + model: 'gpt-4o', + }; + expect(getModelEndUsage({ usage_metadata })).toBe(usage_metadata); + expect(computeUsageCostUSD(usage_metadata, pricing)).toBe(0.0025); +}); + +it('preserves SDK-normalized twelve-hour usage without raw response metadata', () => { + const usage_metadata: UsageMetadata = { + input_tokens: 10000, + input_token_details: { cache_creation: 10000, cache_creation_12h: 10000 }, + model, + provider: Providers.OPENAI, + }; + expect(getModelEndUsage({ usage_metadata })).toBe(usage_metadata); + expect(computeUsageCostUSD(usage_metadata, pricing)).toBe(0.5); +}); diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 7512b7a074..4679913878 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -97,6 +97,12 @@ import { resolveSubagentMaxTurns, resolveRecursionLimit, } from '~/agents/config'; +import { + extractDefaultParams, + getOpenAILLMConfig, + isNativeCyberRequest, + resolveReasoningParams, +} from '~/endpoints/openai/llm'; import { isSteeringSupported, isSteerPreemptSupported, @@ -117,7 +123,6 @@ import { stripIntentFromToolRegistry, stripIntentFromToolDefinitions } from '~/a import { createAgentToolApprovalSession, bindRunToolApprovalSession } from './hitl/modes'; import { resolveConfigHeaders, resolveModelHeaders, mergeHeaders } from '~/utils/headers'; import { interruptToolHandler, supportsRunInterruption } from '~/agents/steering/tools'; -import { extractDefaultParams, resolveReasoningParams } from '~/endpoints/openai/llm'; import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm'; import { reportSubagentResolutionFailure } from '~/agents/failures/subagent'; import { createScheduledMCPRunPolicy } from '~/schedules/authorization/run'; @@ -137,6 +142,7 @@ import { getOpenAIConfig } from '~/endpoints/openai/config'; import { createStepBudgetHook } from '~/agents/stepBudget'; import { buildHITLRunWiring } from '~/agents/hitl/runtime'; import { buildLangfuseConfig } from '~/langfuse/config'; +import { getCustomEndpointConfig } from '~/app/config'; import { applyTestRunHook } from '~/agents/testHook'; import { frameDynamicInstructions } from './tail'; import { isUserProvided } from '~/utils/common'; @@ -678,25 +684,27 @@ function agentModelKwargs( return isPlainObject(modelParameters.modelKwargs) ? modelParameters.modelKwargs : undefined; } -/** - * `getOpenAILLMConfig` carries an Azure Astra deployment alias in - * `modelKwargs.model`, which langchain spreads after `model`. The SDK's - * same-provider summarizer copies the agent's client options and overrides only - * `model`, so a summarizer on another model would still reach the agent's - * deployment. Hand it the agent's kwargs without the alias. - */ +/** The SDK inherits same-provider kwargs. Model switches must drop deployment aliases and Cyber access. */ function summarizationModelKwargs( agentKwargs: Record | undefined, agentModel: string | undefined, summarizationModel: string | undefined, ): Record | undefined { - if (agentKwargs == null || !('model' in agentKwargs)) { + if ( + agentKwargs == null || + !isNonEmptyString(summarizationModel) || + summarizationModel === agentModel + ) { return undefined; } - if (!isNonEmptyString(summarizationModel) || summarizationModel === agentModel) { + const cyberAccess = agentModel === 'gpt-6-cyber' && 'access_programs' in agentKwargs; + if (!('model' in agentKwargs) && !cyberAccess) { return undefined; } const { model: _alias, ...kwargs } = agentKwargs; + if (cyberAccess) { + delete kwargs.access_programs; + } return kwargs; } @@ -770,6 +778,7 @@ function resolveBuiltInClientOverrides( agentProvider?: string; }, appConfig: AppConfig, + user?: IUser, ): SummarizationClientOverrides | undefined { const { model, parameters } = target; /** @@ -803,6 +812,7 @@ function resolveBuiltInClientOverrides( const { llmConfig } = getOpenAIConfig( '', { + requestIdentity: { userId: user?.id, tenantId: user?.tenantId }, modelOptions: { model, reasoning_effort: summarizationReasoningEffort(parameters) }, reverseProxyUrl: baseURL, }, @@ -959,6 +969,7 @@ function resolveOpenAISummarization( const { llmConfig, configOptions } = getOpenAIConfig( apiKey, { + requestIdentity: { userId: headerContext.user?.id, tenantId: headerContext.user?.tenantId }, modelOptions: { model, reasoning_effort: summarizationReasoningEffort(parameters), @@ -1219,7 +1230,12 @@ function resolveSummarizationProvider( if (!customEndpointConfig) { return { provider: overrideProvider, - clientOverrides: resolveBuiltInClientOverrides(overrideProvider, target, appConfig), + clientOverrides: resolveBuiltInClientOverrides( + overrideProvider, + target, + appConfig, + headerContext.user, + ), }; } const rawApiKey = customEndpointConfig.apiKey ?? ''; @@ -1314,6 +1330,8 @@ function resolveSummarizationProvider( apiKey, { reverseProxyUrl: baseURL, + requestIdentity: { userId: headerContext.user?.id, tenantId: headerContext.user?.tenantId }, + ...(target.model === 'gpt-6-cyber' ? { modelOptions: { model: target.model } } : {}), proxy: process.env.PROXY ?? null, headers: resolvedHeaders, addParams: customEndpointConfig.addParams, @@ -1491,6 +1509,79 @@ function shapeSummarizationConfig( parameters: modelKwargs != null ? { modelKwargs, ...mergedParameters } : mergedParameters, }); + if ( + model === 'gpt-6-cyber' && + config?.enabled !== false && + !targetUnavailable && + provider === Providers.OPENAI + ) { + // The SDK inherits source options only for same-provider summaries. + const inherited: Record = { + ...(provider === fallbackProvider ? agent?.model_parameters : undefined), + ...parameters, + }; + const configuration = isPlainObject(inherited.configuration) + ? inherited.configuration + : undefined; + const baseURL = configuration?.baseURL ?? inherited.baseURL; + if (configuration?.baseURL == null && typeof inherited.baseURL === 'string') { + parameters = { + ...parameters, + configuration: { ...configuration, baseURL: inherited.baseURL }, + }; + } + const nativeRequest = { + model, + endpoint: provider, + baseURL: typeof baseURL === 'string' ? baseURL : undefined, + azure: targetsAzure ? (agent?.azureOptions ?? {}) : undefined, + }; + if (isNativeCyberRequest(nativeRequest)) { + const endpointConfig = + clientOverrides == null && provider === fallbackProvider && appConfig && agentEndpoint + ? getCustomEndpointConfig({ endpoint: agentEndpoint, appConfig }) + : undefined; + const endpointDefaults = extractDefaultParams(endpointConfig?.customParams?.paramDefinitions); + const endpointApiMode = endpointConfig?.dropParams?.includes('useResponsesApi') + ? false + : (endpointConfig?.addParams?.useResponsesApi ?? endpointDefaults?.useResponsesApi); + const configuredApiMode = typeof endpointApiMode === 'boolean' ? endpointApiMode : undefined; + const inheritedApiMode = + model === fallbackModel && typeof inherited.useResponsesApi === 'boolean' + ? inherited.useResponsesApi + : undefined; + const summaryApiMode = + typeof parameters?.useResponsesApi === 'boolean' ? parameters.useResponsesApi : undefined; + const { llmConfig } = getOpenAILLMConfig({ + apiKey: '', + streaming: false, + ...nativeRequest, + requestIdentity: { + userId: headerContext.user?.id, + tenantId: headerContext.tenantId ?? headerContext.user?.tenantId, + }, + modelOptions: { + model, + useResponsesApi: summaryApiMode ?? configuredApiMode ?? inheritedApiMode, + }, + addParams: { + ...(isPlainObject(inherited.modelKwargs) ? inherited.modelKwargs : undefined), + model, + }, + }); + parameters = { + ...parameters, + useResponsesApi: llmConfig.useResponsesApi, + modelKwargs: llmConfig.modelKwargs, + user: undefined, + safety_identifier: undefined, + reasoning: undefined, + reasoning_effort: undefined, + include_reasoning: undefined, + }; + } + } + /** The SDK sets maxTokens for this cap, but LangChain spreads modelKwargs after it. */ const parameterTokenCap = userParameters?.maxSummaryTokens; const summaryTokenCap = diff --git a/packages/api/src/agents/transactions.ts b/packages/api/src/agents/transactions.ts index c5a54abc54..c103e99696 100644 --- a/packages/api/src/agents/transactions.ts +++ b/packages/api/src/agents/transactions.ts @@ -15,7 +15,7 @@ interface GetMultiplierParams { interface GetCacheMultiplierParams { /** `write1h` is Anthropic's 1-hour cache write; `write` is the 5-minute rate */ - cacheType: 'write' | 'read' | 'write1h'; + cacheType: 'write' | 'read' | 'write1h' | 'write12h'; model?: string; endpointTokenConfig?: EndpointTokenConfig; inputTokenCount?: number; @@ -50,6 +50,8 @@ interface StructuredTxData extends BaseTxData { writeTokens?: number; /** Subset of `writeTokens` written to the 1-hour cache */ write1hTokens?: number; + /** Subset of `writeTokens` written to the 12-hour cache */ + write12hTokens?: number; readTokens?: number; inputTokenCount?: number; rawAmount?: number; @@ -71,6 +73,8 @@ export interface StructuredPromptTokens { write?: number; /** Subset of `write` written to the 1-hour cache, billed at its own rate */ write1h?: number; + /** Subset of `write` written to the 12-hour cache, billed at its own rate */ + write12h?: number; read?: number; } @@ -151,6 +155,7 @@ function calculateStructuredTokenValue( const inputAbs = Math.abs(txData.inputTokens ?? 0); const writeAbs = Math.abs(txData.writeTokens ?? 0); const write1hAbs = Math.min(Math.abs(txData.write1hTokens ?? 0), writeAbs); + const write12hAbs = Math.min(Math.abs(txData.write12hTokens ?? 0), writeAbs - write1hAbs); const readAbs = Math.abs(txData.readTokens ?? 0); const totalPromptTokens = inputAbs + writeAbs + readAbs; const write1hMultiplier = @@ -163,18 +168,31 @@ function calculateStructuredTokenValue( }) ?? writeMultiplier) : writeMultiplier; + /** Missing TTL pricing retains the existing write/input fallback. */ + const write12hMultiplier = + write12hAbs > 0 + ? (pricing.getCacheMultiplier({ + cacheType: 'write12h', + model, + endpointTokenConfig, + inputTokenCount, + }) ?? writeMultiplier) + : writeMultiplier; + const promptValue = inputAbs * Math.abs(inputMultiplier) + - (writeAbs - write1hAbs) * Math.abs(writeMultiplier) + + (writeAbs - write1hAbs - write12hAbs) * Math.abs(writeMultiplier) + write1hAbs * Math.abs(write1hMultiplier) + + write12hAbs * Math.abs(write12hMultiplier) + readAbs * Math.abs(readMultiplier); const rate = totalPromptTokens > 0 ? promptValue / totalPromptTokens : Math.abs(inputMultiplier); const tokenValue = -( inputAbs * inputMultiplier + - (writeAbs - write1hAbs) * writeMultiplier + + (writeAbs - write1hAbs - write12hAbs) * writeMultiplier + write1hAbs * write1hMultiplier + + write12hAbs * write12hMultiplier + readAbs * readMultiplier ); @@ -187,6 +205,7 @@ function calculateStructuredTokenValue( write: writeMultiplier, read: readMultiplier, ...(write1hAbs > 0 && { write1h: write1hMultiplier }), + ...(write12hAbs > 0 && { write12h: write12hMultiplier }), }, }; } @@ -315,6 +334,7 @@ export function prepareStructuredTokenSpend( const input = Math.max(promptTokens.input ?? 0, 0); const write = Math.max(promptTokens.write ?? 0, 0); const write1h = Math.min(Math.max(promptTokens.write1h ?? 0, 0), write); + const write12h = Math.min(Math.max(promptTokens.write12h ?? 0, 0), write - write1h); const read = Math.max(promptTokens.read ?? 0, 0); const totalInputTokens = input + write + read; const entry = prepareStructuredTx( @@ -324,6 +344,7 @@ export function prepareStructuredTokenSpend( inputTokens: -input, writeTokens: -write, ...(write1h > 0 && { write1hTokens: -write1h }), + ...(write12h > 0 && { write12hTokens: -write12h }), readTokens: -read, inputTokenCount: totalInputTokens, }, diff --git a/packages/api/src/agents/usage.bulk-parity.spec.ts b/packages/api/src/agents/usage.bulk-parity.spec.ts index 68b3408731..667bc397e3 100644 --- a/packages/api/src/agents/usage.bulk-parity.spec.ts +++ b/packages/api/src/agents/usage.bulk-parity.spec.ts @@ -258,7 +258,7 @@ describe('recordCollectedUsage — bulk path parity', () => { it('prices the 1-hour share of the writes at the 1-hour rate in bulk docs', async () => { mockPricing.getMultiplier = jest.fn().mockReturnValue(2); mockPricing.getCacheMultiplier = jest.fn( - ({ cacheType }) => ({ write: 2.5, write1h: 4, read: 0.2 })[cacheType], + ({ cacheType }) => ({ write: 2.5, write1h: 4, write12h: null, read: 0.2 })[cacheType], ); const collectedUsage: UsageMetadata[] = [ { diff --git a/packages/api/src/agents/usage.spec.ts b/packages/api/src/agents/usage.spec.ts index 7dbda10b09..2f1c978be5 100644 --- a/packages/api/src/agents/usage.spec.ts +++ b/packages/api/src/agents/usage.spec.ts @@ -1954,7 +1954,8 @@ describe('computeUsageCostUSD', () => { }, { ...pricing, - getCacheMultiplier: ({ cacheType }) => ({ write: 3.75, write1h: 6, read: 0.3 })[cacheType], + getCacheMultiplier: ({ cacheType }) => + ({ write: 3.75, write1h: 6, write12h: null, read: 0.3 })[cacheType], }, ); expect(cost).toBeCloseTo((1000 * 3 + 500 * 3.75 + 1500 * 6 + 10000 * 0.3 + 500 * 15) / 1e6); diff --git a/packages/api/src/agents/usage.ts b/packages/api/src/agents/usage.ts index 52158b175c..c14ab39bdd 100644 --- a/packages/api/src/agents/usage.ts +++ b/packages/api/src/agents/usage.ts @@ -64,6 +64,10 @@ interface RawProviderUsage { ephemeral_5m_input_tokens?: unknown; ephemeral_1h_input_tokens?: unknown; } | null; + /** OpenAI Responses */ + input_tokens_details?: { + cache_write_tokens_details?: { ttl_12h_tokens?: unknown } | null; + } | null; /** Bedrock Converse */ cacheDetails?: ReadonlyArray<{ ttl?: string; inputTokens?: unknown } | null> | null; } @@ -98,9 +102,9 @@ function getOneHourCacheWrites(usage?: RawProviderUsage | null): number { * Usage of one finished model call. Providers split cache writes by TTL only in * their raw usage (Anthropic's `cache_creation.ephemeral_1h_input_tokens`, Bedrock's * `cacheDetails`), which the agents SDK keeps on `response_metadata` but leaves out - * of `usage_metadata`. The 1-hour count is folded into - * `input_token_details.cache_creation_1h` so those writes are billed at the 1-hour - * rate; without a breakdown the usage is returned unchanged. + * of `usage_metadata`. OpenAI Responses keeps its 12-hour share in + * `input_tokens_details.cache_write_tokens_details.ttl_12h_tokens`. These shares + * are subsets of aggregate writes, not additional tokens. */ export function getModelEndUsage(output?: ModelEndOutput | null): UsageMetadata | undefined { const usage = output?.usage_metadata; @@ -110,14 +114,19 @@ export function getModelEndUsage(output?: ModelEndOutput | null): UsageMetadata const metadata = output?.response_metadata; const oneHour = getOneHourCacheWrites(metadata?.usage) || getOneHourCacheWrites(metadata?.metadata?.usage); - if (!(oneHour > 0)) { + const twelveHour = Number( + metadata?.usage?.input_tokens_details?.cache_write_tokens_details?.ttl_12h_tokens, + ); + if (!(oneHour > 0) && !(twelveHour > 0)) { return usage; } + const writes = getCacheCreationTokens(usage); return { ...usage, input_token_details: { ...usage.input_token_details, - cache_creation_1h: Math.min(oneHour, getCacheCreationTokens(usage)), + ...(oneHour > 0 && { cache_creation_1h: Math.min(oneHour, writes) }), + ...(twelveHour > 0 && { cache_creation_12h: Math.min(twelveHour, writes) }), }, }; } @@ -174,6 +183,8 @@ interface SplitUsage { cacheCreation: number; /** Subset of `cacheCreation` written to the 1-hour cache */ cacheCreation1h: number; + /** Subset of `cacheCreation` written to the 12-hour cache */ + cacheCreation12h: number; cacheRead: number; /** Total prompt tokens including cached portion */ totalInput: number; @@ -198,6 +209,10 @@ export interface CollectedUsageBreakdown { function splitUsage(usage: UsageMetadata): SplitUsage { const cacheCreation = getCacheCreationTokens(usage); const cacheCreation1h = getCacheCreation1hTokens(usage, cacheCreation); + const cacheCreation12h = Math.min( + Math.max(Number(usage.input_token_details?.cache_creation_12h) || 0, 0), + cacheCreation - cacheCreation1h, + ); const cacheRead = Number(usage.input_token_details?.cache_read) || Number(usage.cache_read_input_tokens) || 0; const rawInput = Number(usage.input_tokens) || 0; @@ -207,6 +222,7 @@ function splitUsage(usage: UsageMetadata): SplitUsage { inputOnly: Math.max(0, rawInput - cacheCreation - cacheRead), cacheCreation, cacheCreation1h, + cacheCreation12h, cacheRead, totalInput: rawInput, completion, @@ -216,6 +232,7 @@ function splitUsage(usage: UsageMetadata): SplitUsage { inputOnly: rawInput, cacheCreation, cacheCreation1h, + cacheCreation12h, cacheRead, totalInput: rawInput + cacheCreation + cacheRead, completion, @@ -294,7 +311,8 @@ export function computeUsageCostUSD( pricing: PricingFns, endpointTokenConfig?: EndpointTokenConfig, ): number { - const { inputOnly, cacheCreation, cacheCreation1h, cacheRead, completion } = splitUsage(usage); + const { inputOnly, cacheCreation, cacheCreation1h, cacheCreation12h, cacheRead, completion } = + splitUsage(usage); /** user/context/conversationId only populate the transaction doc, which is * discarded here — only `tokenValue` (credits) is summed */ const txData: TxMetadata = { @@ -313,6 +331,7 @@ export function computeUsageCostUSD( input: inputOnly, write: cacheCreation, ...(cacheCreation1h > 0 && { write1h: cacheCreation1h }), + ...(cacheCreation12h > 0 && { write12h: cacheCreation12h }), read: cacheRead, }, completionTokens: completion, @@ -953,7 +972,7 @@ export async function recordCollectedUsage( continue; } - const { inputOnly, cacheCreation, cacheCreation1h, cacheRead, completion } = + const { inputOnly, cacheCreation, cacheCreation1h, cacheCreation12h, cacheRead, completion } = splitUsage(usage); if (options?.excludeFromOutputTotal !== true) { @@ -986,6 +1005,7 @@ export async function recordCollectedUsage( input: inputOnly, write: cacheCreation, ...(cacheCreation1h > 0 && { write1h: cacheCreation1h }), + ...(cacheCreation12h > 0 && { write12h: cacheCreation12h }), read: cacheRead, }, completionTokens: completion, @@ -1011,6 +1031,7 @@ export async function recordCollectedUsage( input: inputOnly, write: cacheCreation, ...(cacheCreation1h > 0 && { write1h: cacheCreation1h }), + ...(cacheCreation12h > 0 && { write12h: cacheCreation12h }), read: cacheRead, }, completionTokens: completion, diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.ts index a7b0f22605..3a1b2fe341 100644 --- a/packages/api/src/endpoints/custom/initialize.ts +++ b/packages/api/src/endpoints/custom/initialize.ts @@ -365,6 +365,7 @@ export async function initializeCustom( } else { const finalClientOptions = { ...clientOptions, + requestIdentity: { userId, tenantId }, modelOptions, inferenceParameters, }; diff --git a/packages/api/src/endpoints/openai/config.ts b/packages/api/src/endpoints/openai/config.ts index 19455f2979..bf6ce942a3 100644 --- a/packages/api/src/endpoints/openai/config.ts +++ b/packages/api/src/endpoints/openai/config.ts @@ -209,6 +209,7 @@ export function getOpenAIConfig( inferenceParameters, modelOptions, useOpenRouter, + requestIdentity: options.requestIdentity, reasoningFormat: getReasoningFormat({ customFormat: options.customParams?.reasoningFormat, isVercel: Boolean(isVercel), diff --git a/packages/api/src/endpoints/openai/cyber.spec.ts b/packages/api/src/endpoints/openai/cyber.spec.ts new file mode 100644 index 0000000000..4486e38ce0 --- /dev/null +++ b/packages/api/src/endpoints/openai/cyber.spec.ts @@ -0,0 +1,436 @@ +import { createHash } from 'node:crypto'; +import { HumanMessage } from '@librechat/agents/langchain'; +import { Providers, initializeModel } from '@librechat/agents'; +import { EModelEndpoint, ReasoningEffort } from 'librechat-data-provider'; +import type { OpenAIConfiguration, BaseInitializeParams } from '~/types'; +import { initializeOpenAI } from './initialize'; +import { getOpenAILLMConfig } from './llm'; +import { getOpenAIConfig } from './config'; + +const model = 'gpt-6-cyber'; +const identity = { userId: 'authenticated-user', tenantId: 'tenant-a' }; +const safetyId = (userId = identity.userId, tenantId = identity.tenantId) => + createHash('sha256') + .update(JSON.stringify(['librechat:gpt-6-cyber:v1', tenantId, userId])) + .digest('hex'); +const config = (overrides: Partial[0]> = {}) => + getOpenAILLMConfig({ + apiKey: 'test-key', + streaming: false, + endpoint: EModelEndpoint.openAI, + modelOptions: { model }, + requestIdentity: identity, + ...overrides, + }).llmConfig; + +it('defaults approved Cyber requests to Responses without a guessed reasoning effort', () => { + expect(config()).toMatchObject({ + useResponsesApi: true, + modelKwargs: { + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: safetyId(), + }, + }); + expect(config()).not.toHaveProperty('reasoning'); + expect( + config({ modelOptions: { model, reasoning_effort: ReasoningEffort.high } }), + ).not.toHaveProperty('reasoning'); +}); + +it('derives identity from the authenticated principal rather than overrideable wire parameters', () => { + const addParams = { user: 'spoofed', safety_identifier: 'shared-literal' }; + const result = config({ modelOptions: { model, user: 'request-user' }, addParams }); + expect(result).not.toHaveProperty('user'); + expect(result.modelKwargs).not.toHaveProperty('user'); + expect(result.modelKwargs?.safety_identifier).toBe(safetyId()); + expect(config().modelKwargs?.safety_identifier).toBe(result.modelKwargs?.safety_identifier); + expect( + config({ requestIdentity: { ...identity, userId: 'another-user' } }).modelKwargs + ?.safety_identifier, + ).not.toBe(safetyId()); + expect( + config({ requestIdentity: { ...identity, tenantId: 'tenant-b' } }).modelKwargs + ?.safety_identifier, + ).not.toBe(safetyId()); + expect(addParams).toEqual({ user: 'spoofed', safety_identifier: 'shared-literal' }); +}); + +it.each([{}, { userId: '' }, { userId: ' ' }])( + 'rejects missing authenticated identity: %j', + (requestIdentity) => { + expect(() => config({ requestIdentity })).toThrow( + 'GPT-6 Cyber requires an authenticated user.', + ); + }, +); + +it.each(['modelKwargs', 'access_programs', 'safety_identifier'])( + 'rejects removal of required wire fields: %s', + (field) => { + expect(() => config({ dropParams: [field] })).toThrow( + 'GPT-6 Cyber requires its access program and safety identifier.', + ); + }, +); + +it('preserves explicit administrator access-program and routing selections', () => { + const access_programs = { cyber: 'standard' }; + const result = config({ addParams: { access_programs, useResponsesApi: false } }); + expect(result.modelKwargs?.access_programs).toEqual(access_programs); + expect(result.useResponsesApi).toBe(false); +}); + +it.each([ + { modelOptions: { model: 'gpt-6-sol' } }, + { modelOptions: { model: 'gpt-6-cyber-other' } }, + { baseURL: 'https://gateway.example/v1' }, + { baseURL: 'https://api.openai.com.evil.example/v1' }, + { useOpenRouter: true }, + { endpoint: EModelEndpoint.azureOpenAI }, +])('leaves unrelated models and transports unchanged: %j', (overrides) => { + expect(config(overrides).modelKwargs ?? {}).not.toHaveProperty('access_programs'); + expect(config(overrides).modelKwargs ?? {}).not.toHaveProperty('safety_identifier'); +}); + +it('recognizes a custom endpoint pointed directly at OpenAI, using the final model override', () => { + expect( + config({ endpoint: 'Daybreak', baseURL: 'https://api.openai.com/v1' }).modelKwargs + ?.safety_identifier, + ).toBe(safetyId()); + expect(config({ modelOptions: { model: 'gpt-4.1' }, addParams: { model } }).useResponsesApi).toBe( + true, + ); + expect(config({ addParams: { model: 'gpt-4.1' } }).modelKwargs ?? {}).not.toHaveProperty( + 'safety_identifier', + ); +}); + +it.each([false, true])( + 'serializes the required Cyber wire body with tools and images (streaming=%s)', + async (streaming) => { + const bodies: Record[] = []; + const urls: string[] = []; + const response = { + id: 'resp_test', + object: 'response', + status: 'completed', + model, + output: [ + { + type: 'message', + id: 'msg_test', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'OK', annotations: [] }], + }, + ], + usage: { + input_tokens: 300000, + output_tokens: 100, + total_tokens: 300100, + input_tokens_details: { cached_tokens: 100000 }, + output_tokens_details: { reasoning_tokens: 10 }, + }, + }; + const fetch: NonNullable['fetch']> = async (url, init) => { + urls.push(String(url)); + bodies.push(JSON.parse(String(init?.body))); + if (!streaming) return Response.json(response); + const events = [ + { type: 'response.created', response: { ...response, status: 'in_progress', output: [] } }, + { + type: 'response.output_text.delta', + item_id: 'msg_test', + output_index: 0, + content_index: 0, + delta: 'OK', + }, + { type: 'response.completed', response }, + ]; + return new Response( + events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(''), + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + }; + const { llmConfig, configOptions } = getOpenAIConfig( + 'test-key', + { + streaming, + requestIdentity: identity, + modelOptions: { model, max_tokens: 128000, reasoning_effort: ReasoningEffort.high }, + }, + EModelEndpoint.openAI, + ); + const llm = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { + ...llmConfig, + verbosity: undefined, + streamUsage: true, + configuration: { ...configOptions, fetch }, + }, + tools: [ + { + type: 'function', + function: { + name: 'verify_patch', + description: 'Verify a fix', + parameters: { type: 'object', properties: {} }, + }, + }, + ], + }); + const messages = [ + new HumanMessage({ + content: [ + { type: 'text', text: 'Verify the fix' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,dGVzdA==' } }, + ], + }), + ]; + if (streaming) { + for await (const chunk of await llm.stream(messages)) expect(chunk).toBeDefined(); + } else { + expect((await llm.invoke(messages)).usage_metadata?.input_token_details?.cache_read).toBe( + 100000, + ); + } + expect(urls).toEqual(['https://api.openai.com/v1/responses']); + expect(bodies).toHaveLength(1); + expect(bodies[0]).toMatchObject({ + model, + stream: streaming, + max_output_tokens: 128000, + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: safetyId(), + tools: [expect.objectContaining({ type: 'function', name: 'verify_patch' })], + }); + expect(JSON.stringify(bodies[0].input)).toContain('data:image/png;base64,dGVzdA=='); + for (const key of [ + 'user', + 'reasoning', + 'reasoning_effort', + 'max_tokens', + 'max_completion_tokens', + ]) + expect(bodies[0]).not.toHaveProperty(key); + }, +); + +it('takes identity from the authenticated runtime, not submitted model fields', async () => { + const previous = process.env.OPENAI_API_KEY; + const previousURL = process.env.OPENAI_REVERSE_PROXY; + process.env.OPENAI_API_KEY = 'test-key'; + delete process.env.OPENAI_REVERSE_PROXY; + const params = { + req: { + user: { id: identity.userId, tenantId: identity.tenantId }, + body: {}, + config: {}, + } as BaseInitializeParams['req'], + endpoint: EModelEndpoint.openAI, + model_parameters: { + model, + user: 'spoofed-user', + safety_identifier: 'shared-literal', + requestIdentity: { userId: 'spoofed' }, + }, + db: { getUserKey: jest.fn(), getUserKeyValues: jest.fn() }, + }; + try { + const result = await initializeOpenAI(params); + expect(result.llmConfig).toHaveProperty('modelKwargs.safety_identifier', safetyId()); + expect(params.db.getUserKeyValues).not.toHaveBeenCalled(); + } finally { + if (previous === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previous; + if (previousURL === undefined) delete process.env.OPENAI_REVERSE_PROXY; + else process.env.OPENAI_REVERSE_PROXY = previousURL; + } +}); + +describe('Cyber SDK-effective URL routing', () => { + beforeEach(() => { + const env: NodeJS.ProcessEnv = { ...process.env, OPENAI_API_KEY: 'test-key' }; + for (const key of [ + 'OPENAI_API_BASE', + 'OPENAI_BASE_URL', + 'OPENAI_REVERSE_PROXY', + 'LANGSMITH_GATEWAY', + ]) { + delete env[key]; + } + jest.replaceProperty(process, 'env', env); + }); + + afterEach(() => jest.restoreAllMocks()); + + it.each([ + { + apiBase: undefined, + sdkURL: undefined, + reverseProxy: undefined, + gateway: undefined, + origin: 'https://api.openai.com', + native: true, + }, + { + apiBase: undefined, + sdkURL: 'https://sdk-gateway.example/v1', + reverseProxy: undefined, + gateway: undefined, + origin: 'https://sdk-gateway.example', + native: false, + }, + { + apiBase: 'https://api-base-gateway.example/v1', + sdkURL: undefined, + reverseProxy: undefined, + gateway: undefined, + origin: 'https://api-base-gateway.example', + native: false, + }, + { + apiBase: 'https://api-base-gateway.example/v1', + sdkURL: 'https://api.openai.com/v1', + reverseProxy: undefined, + gateway: undefined, + origin: 'https://api-base-gateway.example', + native: false, + }, + { + apiBase: 'https://api.openai.com/v1', + sdkURL: 'https://sdk-gateway.example/v1', + reverseProxy: undefined, + gateway: undefined, + origin: 'https://api.openai.com', + native: true, + }, + { + apiBase: '', + sdkURL: 'https://sdk-gateway.example/v1', + reverseProxy: undefined, + gateway: undefined, + origin: 'https://sdk-gateway.example', + native: false, + }, + { + apiBase: 'https://api-base-gateway.example/v1', + sdkURL: 'https://sdk-gateway.example/v1', + reverseProxy: 'https://api.openai.com/v1', + gateway: undefined, + origin: 'https://api.openai.com', + native: true, + }, + { + apiBase: 'https://api.openai.com/v1', + sdkURL: undefined, + reverseProxy: 'https://explicit-gateway.example/v1', + gateway: undefined, + origin: 'https://explicit-gateway.example', + native: false, + }, + { + apiBase: undefined, + sdkURL: undefined, + reverseProxy: undefined, + gateway: 'true', + origin: 'https://gateway.smith.langchain.com', + native: false, + }, + { + apiBase: undefined, + sdkURL: undefined, + reverseProxy: undefined, + gateway: 'false', + origin: 'https://api.openai.com', + native: true, + }, + { + apiBase: undefined, + sdkURL: undefined, + reverseProxy: 'https://api.openai.com/v1', + gateway: 'true', + origin: 'https://api.openai.com', + native: true, + }, + ])( + 'matches the published SDK endpoint without native-field leakage: %j', + async ({ apiBase, sdkURL, reverseProxy, gateway, origin, native }) => { + for (const [key, value] of Object.entries({ + OPENAI_API_BASE: apiBase, + OPENAI_BASE_URL: sdkURL, + OPENAI_REVERSE_PROXY: reverseProxy, + LANGSMITH_GATEWAY: gateway, + })) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + const options = await initializeOpenAI({ + req: { + user: { id: identity.userId, tenantId: identity.tenantId }, + body: {}, + config: {}, + } as BaseInitializeParams['req'], + endpoint: EModelEndpoint.openAI, + model_parameters: { model }, + db: { getUserKey: jest.fn(), getUserKeyValues: jest.fn() }, + }); + const llmConfig = options.llmConfig as ReturnType['llmConfig']; + const requests: { url: URL; body: Record }[] = []; + const fetch: NonNullable['fetch']> = async (url, init) => { + requests.push({ url: new URL(String(url)), body: JSON.parse(String(init?.body)) }); + return Response.json({ + id: 'resp_url', + object: 'response', + status: 'completed', + model, + choices: [ + { index: 0, message: { role: 'assistant', content: 'OK' }, finish_reason: 'stop' }, + ], + output: [ + { + type: 'message', + id: 'msg_url', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'OK', annotations: [] }], + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 2, + input_tokens: 10, + output_tokens: 2, + total_tokens: 12, + }, + }); + }; + const llm = initializeModel({ + provider: Providers.OPENAI, + clientOptions: { + ...llmConfig, + verbosity: undefined, + streaming: false, + configuration: { ...options.configOptions, fetch }, + }, + }); + await llm.invoke('Check endpoint routing'); + expect(requests).toHaveLength(1); + expect(requests[0].url.origin).toBe(origin); + expect(requests[0].url.pathname.endsWith(native ? '/responses' : '/chat/completions')).toBe( + true, + ); + if (native) { + expect(requests[0].body).toMatchObject({ + access_programs: { cyber: 'daybreak_red' }, + safety_identifier: safetyId(), + }); + } else { + expect(requests[0].body).not.toHaveProperty('access_programs'); + expect(requests[0].body).not.toHaveProperty('safety_identifier'); + expect(llmConfig.useResponsesApi).not.toBe(true); + } + }, + ); +}); diff --git a/packages/api/src/endpoints/openai/initialize.ts b/packages/api/src/endpoints/openai/initialize.ts index 49d15520b9..b845759e1e 100644 --- a/packages/api/src/endpoints/openai/initialize.ts +++ b/packages/api/src/endpoints/openai/initialize.ts @@ -195,6 +195,7 @@ export async function initializeOpenAI( const finalClientOptions: OpenAIConfigOptions = { ...clientOptions, + requestIdentity: { userId: user?.id, tenantId: user?.tenantId }, modelOptions, }; diff --git a/packages/api/src/endpoints/openai/llm.ts b/packages/api/src/endpoints/openai/llm.ts index 1b9f83225c..2477b8c4d3 100644 --- a/packages/api/src/endpoints/openai/llm.ts +++ b/packages/api/src/endpoints/openai/llm.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { Providers, isOpenAILike } from '@librechat/agents'; import { EModelEndpoint, @@ -184,6 +185,36 @@ function isCanonicalOpenAIBaseURL(baseURL?: string | null): boolean { } } +export function isNativeCyberRequest({ + model, + endpoint, + baseURL, + azure, + useOpenRouter, +}: { + model?: string; + endpoint?: string | null; + baseURL?: string | null; + azure?: false | t.AzureOptions; + useOpenRouter?: boolean; +}): boolean { + if ( + model !== 'gpt-6-cyber' || + useOpenRouter || + azure || + (endpoint !== EModelEndpoint.openAI && baseURL == null) + ) { + return false; + } + // Agents 4.2.2 resolves explicit URLs, API_BASE, BASE_URL, then the SDK gateway. + const effectiveURL = baseURL || process.env.OPENAI_API_BASE || process.env.OPENAI_BASE_URL; + if (effectiveURL) { + return isCanonicalOpenAIBaseURL(effectiveURL); + } + const gateway = process.env.LANGSMITH_GATEWAY; + return !gateway || /^(false|0|no)$/i.test(gateway); +} + function removeReasoningField(target: Record, field: string) { const { reasoning } = target; if (reasoning == null || typeof reasoning !== 'object' || Array.isArray(reasoning)) { @@ -609,11 +640,13 @@ export function getOpenAILLMConfig({ defaultParams, inferenceParameters, useOpenRouter, + requestIdentity, reasoningFormat = ReasoningParameterFormat.reasoningEffort, modelOptions: _modelOptions, }: { apiKey: string; streaming: boolean; + requestIdentity?: t.OpenAIConfigOptions['requestIdentity']; baseURL?: string | null; endpoint?: EModelEndpoint | string | null; modelOptions: Partial; @@ -941,16 +974,24 @@ export function getOpenAILLMConfig({ */ const responsesApiExplicitlyOptedOut = dropParams != null && dropParams.includes('useResponsesApi'); + const cyber = isNativeCyberRequest({ + model: llmConfig.model, + useOpenRouter, + azure, + endpoint, + baseURL, + }); const firstPartyOpenAI = - !useOpenRouter && endpoint === EModelEndpoint.openAI && isCanonicalOpenAIBaseURL(baseURL); + !useOpenRouter && + endpoint === EModelEndpoint.openAI && + (llmConfig.model === 'gpt-6-cyber' ? cyber : isCanonicalOpenAIBaseURL(baseURL)); const firstPartyAzure = !useOpenRouter && endpoint === EModelEndpoint.azureOpenAI && isCanonicalAzureBaseURL(baseURL, azure); const firstPartyEndpoint = firstPartyOpenAI || firstPartyAzure; - /** Keep GPT-6 model identity on Azure, with the deployment name used on the wire. */ const firstPartyResponsesModel = - firstPartyEndpoint && prefersResponsesApiByModel(llmConfig.model); + (firstPartyEndpoint && prefersResponsesApiByModel(llmConfig.model)) || cyber; if ( firstPartyOpenAI && reasoningFormat !== ReasoningParameterFormat.disabled && @@ -1113,6 +1154,42 @@ export function getOpenAILLMConfig({ } } + if (cyber) { + if ( + requestIdentity != null && + dropParams?.some((key) => + ['modelKwargs', 'access_programs', 'safety_identifier'].includes(key), + ) + ) { + throw new Error('GPT-6 Cyber requires its access program and safety identifier.'); + } + // No published effort enum: omit restored/user reasoning and leave the provider default. + removeReasoningPayload(llmConfig as Record); + removeReasoningPayload(modelKwargs); + if (modelKwargs.access_programs === undefined) { + modelKwargs.access_programs = { cyber: 'daybreak_red' }; + } + if (requestIdentity != null) { + if (!requestIdentity.userId?.trim()) { + throw new Error('GPT-6 Cyber requires an authenticated user.'); + } + // Required identity is server-owned, never an administrator's shared literal or a request field. + delete (llmConfig as Record).safety_identifier; + delete llmConfig.user; + delete modelKwargs.user; + modelKwargs.safety_identifier = createHash('sha256') + .update( + JSON.stringify([ + 'librechat:gpt-6-cyber:v1', + requestIdentity.tenantId ?? '', + requestIdentity.userId, + ]), + ) + .digest('hex'); + } + hasModelKwargs = true; + } + hasModelKwargs = applyResponsesVerbosity({ llmConfig, diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts index e1f95efc5f..c8b5bf2717 100644 --- a/packages/api/src/stream/interfaces/IJobStore.ts +++ b/packages/api/src/stream/interfaces/IJobStore.ts @@ -866,6 +866,8 @@ export interface UsageMetadata { cache_creation?: number; /** Subset of `cache_creation` written to Anthropic's 1-hour cache (see `getModelEndUsage`) */ cache_creation_1h?: number; + /** Subset of `cache_creation` written to OpenAI's 12-hour cache */ + cache_creation_12h?: number; /** Tokens read from cache */ cache_read?: number; /** OpenAI GPT-5.6+ cache-write tokens (billed above the input rate) */ diff --git a/packages/api/src/types/openai.ts b/packages/api/src/types/openai.ts index 3d96716cd0..90807ed176 100644 --- a/packages/api/src/types/openai.ts +++ b/packages/api/src/types/openai.ts @@ -13,6 +13,8 @@ export type OpenAIModelOptions = Partial; * Configuration options for the getLLMConfig function */ export interface OpenAIConfigOptions { + /** Server-authenticated principal. Omitted only for request-free policy discovery. */ + requestIdentity?: { userId?: string; tenantId?: string }; /** Validated extra inference wire fields. */ inferenceParameters?: Record; modelOptions?: OpenAIModelOptions; diff --git a/packages/api/src/utils/tokens.spec.ts b/packages/api/src/utils/tokens.spec.ts index 3dc4b37dfa..6150a40a4c 100644 --- a/packages/api/src/utils/tokens.spec.ts +++ b/packages/api/src/utils/tokens.spec.ts @@ -499,3 +499,8 @@ describe('Claude Mythos version boundaries', () => { }, ); }); +describe('GPT-6 Cyber', () => { + it('resolves the published context window for configured models', () => { + expect(getModelMaxTokens('gpt-6-cyber', EModelEndpoint.openAI)).toBe(1050000); + }); +}); diff --git a/packages/api/src/utils/tokens.ts b/packages/api/src/utils/tokens.ts index aa6921c0d8..48a6437576 100644 --- a/packages/api/src/utils/tokens.ts +++ b/packages/api/src/utils/tokens.ts @@ -64,6 +64,7 @@ const openAIModels = { 'gpt-5.6': 1050000, 'gpt-5.6-terra': 1050000, 'gpt-5.6-luna': 1050000, + 'gpt-6-cyber': 1050000, 'gpt-6-astra': 1050000, // >272K input prices at the long-context tier (2x input/cache, 1.5x output) 'gpt-6-sol': 1050000, // >272K input prices at the long-context tier (2x input/cache, 1.5x output) 'gpt-6.1-terra': 1050000, diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index a4594ca454..2c7d2fe3c6 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -5131,6 +5131,8 @@ const responsesOnlyOpenAIModels = ['gpt-6-astra']; const responsesReasoningOpenAIModels = ['gpt-6.1-sol', 'gpt-6-sol', 'gpt-6.1-terra', 'gpt-6-luna']; /** Catalog models whose native execution defaults to the Responses API. */ export const responsesPreferredOpenAIModels: readonly string[] = [ + // Restricted to approved Daybreak projects, never a universal catalog default. + 'gpt-6-cyber', ...responsesOnlyOpenAIModels, ...responsesReasoningOpenAIModels, ]; diff --git a/packages/data-provider/src/cyber.spec.ts b/packages/data-provider/src/cyber.spec.ts new file mode 100644 index 0000000000..185f67ef7f --- /dev/null +++ b/packages/data-provider/src/cyber.spec.ts @@ -0,0 +1,30 @@ +import { paramSettings, applyModelAwareDefaults } from './parameterSettings'; +import { defaultModels, initialModelsConfig } from './config'; +import { EModelEndpoint } from './schemas'; + +it('keeps the restricted model out of universal and Assistants catalogs', () => { + for (const models of [...Object.values(defaultModels), ...Object.values(initialModelsConfig)]) { + expect(models).not.toContain('gpt-6-cyber'); + } +}); + +it('offers the published output limit without an invented reasoning effort enum', () => { + const result = applyModelAwareDefaults( + paramSettings[EModelEndpoint.openAI]!, + EModelEndpoint.openAI, + 'gpt-6-cyber', + { 'gpt-6-cyber': { default: true, on: true, off: false } }, + ); + expect(result.find((setting) => setting.key === 'max_tokens')?.range?.max).toBe(128000); + expect(result.find((setting) => setting.key === 'useResponsesApi')?.default).toBe(true); + expect(result.some((setting) => setting.key.startsWith('reasoning_'))).toBe(false); +}); + +it('preserves the actual transport route and administrator custom reasoning definitions', () => { + const settings = paramSettings[EModelEndpoint.openAI]!; + const native = applyModelAwareDefaults(settings, EModelEndpoint.openAI, 'gpt-6-cyber', { + 'gpt-6-cyber': { default: false, on: false, off: false }, + }); + expect(native.find((setting) => setting.key === 'useResponsesApi')?.default).toBe(false); + expect(applyModelAwareDefaults(settings, 'Daybreak', 'gpt-6-cyber')).toBe(settings); +}); diff --git a/packages/data-provider/src/parameterSettings.ts b/packages/data-provider/src/parameterSettings.ts index ab220ac630..8e8cc490bd 100644 --- a/packages/data-provider/src/parameterSettings.ts +++ b/packages/data-provider/src/parameterSettings.ts @@ -1764,6 +1764,37 @@ function applyModelFamilyDefaults( : setting, ); } + if (model === 'gpt-6-cyber' && endpoint === EModelEndpoint.openAI) { + return settings + .filter( + (setting) => + ![ + 'reasoning_effort', + 'reasoning_summary', + 'reasoning_mode', + 'reasoning_context', + ].includes(setting.key), + ) + .map((setting) => { + if (setting.key === 'max_tokens') + return { ...setting, range: { min: 1, max: 128000, step: 1 } }; + if (setting.key === 'useResponsesApi' && endpoint === EModelEndpoint.openAI) { + const route = (value?: boolean) => + resolveEffectiveUseResponsesApi({ + endpoint, + model, + routing: responsesApiRouting, + value, + }); + return { + ...setting, + default: route() ?? false, + enumMappings: { true: route(true) ?? true, false: route(false) ?? false }, + }; + } + return setting; + }); + } const tier = gpt6Tier(model); if (tier === 'sol' || tier === 'terra' || tier === 'luna') { return settings.map((setting) => { diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.ts index 8ee3a83c8f..0cd100553a 100644 --- a/packages/data-provider/src/types/runs.ts +++ b/packages/data-provider/src/types/runs.ts @@ -335,6 +335,8 @@ export type TTokenUsageEvent = { cache_creation?: number; /** Subset of `cache_creation` written to Anthropic's 1-hour cache */ cache_creation_1h?: number; + /** Subset of `cache_creation` written to OpenAI's 12-hour cache */ + cache_creation_12h?: number; cache_read?: number; }; model?: string; diff --git a/packages/data-schemas/src/methods/cache.spec.ts b/packages/data-schemas/src/methods/cache.spec.ts new file mode 100644 index 0000000000..c6110cf4d2 --- /dev/null +++ b/packages/data-schemas/src/methods/cache.spec.ts @@ -0,0 +1,62 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { ITransaction } from '~/schema/transaction'; +import type { IBalance } from '~/types'; +import { matchModelName, findMatchingPattern } from './test-helpers'; +import { createSpendTokensMethods } from './spendTokens'; +import { createTransactionMethods } from './transaction'; +import transactionSchema from '~/schema/transaction'; +import balanceSchema from '~/schema/balance'; +import { createTxMethods } from './tx'; + +let server: MongoMemoryServer; +let Balance: mongoose.Model; +let Transaction: mongoose.Model; +const pricing = createTxMethods(mongoose, { matchModelName, findMatchingPattern }); +const methods = createTransactionMethods(mongoose, pricing); +const { spendStructuredTokens } = createSpendTokensMethods(mongoose, methods); + +beforeAll(async () => { + server = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } }); + await mongoose.connect(server.getUri()); + Balance = mongoose.model('Balance', balanceSchema); + Transaction = mongoose.model('Transaction', transactionSchema); +}, 60000); + +afterAll(async () => { + await mongoose.disconnect(); + await server?.stop(); +}); + +it.each([ + { input: 0, write: 10000, write12h: 10000, read: 0, debit: 500000 }, + { input: 1000, write: 7000, write12h: 4000, read: 4000, debit: 303000 }, + { input: 261000, write: 7000, write12h: 4000, read: 4000, debit: 5503000 }, + { input: 261001, write: 7000, write12h: 4000, read: 4000, debit: 10736040 }, + { input: 0, write: 10000, write12h: 0, read: 0, debit: 250000 }, + { input: 0, write: 10000, write12h: 20000, read: 0, debit: 500000 }, +])( + 'debits the legacy transaction path once for TTL shares: %j', + async ({ debit, ...promptTokens }) => { + const user = new mongoose.Types.ObjectId().toString(); + const initial = 20000000; + await Balance.create({ user, tokenCredits: initial }); + const result = await spendStructuredTokens( + { + user, + model: 'gpt-6-cyber', + conversationId: 'cache-conversation', + balance: { enabled: true }, + }, + { promptTokens }, + ); + expect(result.prompt?.prompt).toBe(-debit); + const transaction = await Transaction.findOne({ user }).lean(); + expect(transaction).toMatchObject({ + tokenValue: -debit, + rawAmount: -(promptTokens.input + promptTokens.write + promptTokens.read), + writeTokens: -promptTokens.write, + }); + expect((await Balance.findOne({ user }).lean())?.tokenCredits).toBe(initial - debit); + }, +); diff --git a/packages/data-schemas/src/methods/cyber.spec.ts b/packages/data-schemas/src/methods/cyber.spec.ts new file mode 100644 index 0000000000..a71c9f1731 --- /dev/null +++ b/packages/data-schemas/src/methods/cyber.spec.ts @@ -0,0 +1,38 @@ +import { createTxMethods, cacheTokenValues, premiumCacheTokenValues } from './tx'; +import { matchModelName, findMatchingPattern } from './test-helpers'; + +const { getMultiplier, getCacheMultiplier } = createTxMethods({} as typeof import('mongoose'), { + matchModelName, + findMatchingPattern, +}); +const model = 'gpt-6-cyber'; + +it.each([272000, 272001])( + 'uses the published whole-request Cyber tier for %i input tokens', + (inputTokenCount) => { + const long = inputTokenCount > 272000; + expect(getMultiplier({ model, tokenType: 'prompt', inputTokenCount })).toBe(long ? 40 : 20); + expect(getMultiplier({ model, tokenType: 'completion', inputTokenCount })).toBe( + long ? 150 : 100, + ); + expect(getCacheMultiplier({ model, cacheType: 'read', inputTokenCount })).toBe(long ? 4 : 2); + expect(getCacheMultiplier({ model, cacheType: 'write', inputTokenCount })).toBe( + long ? null : 25, + ); + expect(getCacheMultiplier({ model, cacheType: 'write12h', inputTokenCount })).toBe( + long ? null : 50, + ); + }, +); + +it('records 30-minute and 12-hour writes without inventing long-context cache-write rates', () => { + expect(cacheTokenValues[model]).toEqual({ write: 25, write12h: 50, read: 2 }); + expect(premiumCacheTokenValues[model]).toEqual({ threshold: 272000, read: 4 }); + expect( + getCacheMultiplier({ valueKey: model, cacheType: 'write', inputTokenCount: 300000 }), + ).toBeNull(); +}); + +it('does not invent an Anthropic-style one-hour cache-write rate for Cyber', () => { + expect(getCacheMultiplier({ model, cacheType: 'write1h' })).toBeNull(); +}); diff --git a/packages/data-schemas/src/methods/spendTokens.ts b/packages/data-schemas/src/methods/spendTokens.ts index e4a6ffb62c..30bf93302d 100644 --- a/packages/data-schemas/src/methods/spendTokens.ts +++ b/packages/data-schemas/src/methods/spendTokens.ts @@ -27,7 +27,13 @@ export function createSpendTokensMethods( spendStructuredTokens: ( txData: SpendTxData, tokenUsage: { - promptTokens?: { input?: number; write?: number; write1h?: number; read?: number }; + promptTokens?: { + input?: number; + write?: number; + write1h?: number; + write12h?: number; + read?: number; + }; completionTokens?: number; }, ) => Promise<{ @@ -93,7 +99,13 @@ export function createSpendTokensMethods( async function spendStructuredTokens( txData: SpendTxData, tokenUsage: { - promptTokens?: { input?: number; write?: number; write1h?: number; read?: number }; + promptTokens?: { + input?: number; + write?: number; + write1h?: number; + write12h?: number; + read?: number; + }; completionTokens?: number; }, ): Promise<{ @@ -113,6 +125,7 @@ export function createSpendTokensMethods( const input = Math.max(promptTokens.input ?? 0, 0); const write = Math.max(promptTokens.write ?? 0, 0); const write1h = Math.min(Math.max(promptTokens.write1h ?? 0, 0), write); + const write12h = Math.min(Math.max(promptTokens.write12h ?? 0, 0), write - write1h); const read = Math.max(promptTokens.read ?? 0, 0); const totalInputTokens = input + write + read; prompt = await transactionMethods.createStructuredTransaction({ @@ -121,6 +134,7 @@ export function createSpendTokensMethods( inputTokens: -input, writeTokens: -write, ...(write1h > 0 && { write1hTokens: -write1h }), + ...(write12h > 0 && { write12hTokens: -write12h }), readTokens: -read, inputTokenCount: totalInputTokens, }); diff --git a/packages/data-schemas/src/methods/transaction.ts b/packages/data-schemas/src/methods/transaction.ts index 57df98cd4d..5e2ce325ac 100644 --- a/packages/data-schemas/src/methods/transaction.ts +++ b/packages/data-schemas/src/methods/transaction.ts @@ -30,7 +30,7 @@ type MultiplierParams = { }; type CacheMultiplierParams = { - cacheType?: 'write' | 'read' | 'write1h'; + cacheType?: 'write' | 'read' | 'write1h' | 'write12h'; model?: string; endpointTokenConfig?: Record>; inputTokenCount?: number; @@ -52,6 +52,8 @@ interface InternalTxDoc { writeTokens?: number; /** Subset of `writeTokens` written to the 1-hour cache */ write1hTokens?: number; + /** Subset of `writeTokens` written to the 12-hour cache */ + write12hTokens?: number; readTokens?: number; } @@ -70,6 +72,8 @@ export interface TxData { writeTokens?: number; /** Subset of `writeTokens` written to the 1-hour cache */ write1hTokens?: number; + /** Subset of `writeTokens` written to the 12-hour cache */ + write12hTokens?: number; readTokens?: number; balance?: Partial; transactions?: { enabled?: boolean }; @@ -201,6 +205,7 @@ export function createTransactionMethods( const inputAbs = Math.abs(txn.inputTokens ?? 0); const writeAbs = Math.abs(txn.writeTokens ?? 0); const write1hAbs = Math.min(Math.abs(txn.write1hTokens ?? 0), writeAbs); + const write12hAbs = Math.min(Math.abs(txn.write12hTokens ?? 0), writeAbs - write1hAbs); const readAbs = Math.abs(txn.readTokens ?? 0); const write1hMultiplier = write1hAbs > 0 @@ -212,18 +217,31 @@ export function createTransactionMethods( }) ?? writeMultiplier) : writeMultiplier; + /** Missing TTL pricing retains the existing write/input fallback. */ + const write12hMultiplier = + write12hAbs > 0 + ? (txMethods.getCacheMultiplier({ + cacheType: 'write12h', + model, + endpointTokenConfig: etConfig, + inputTokenCount, + }) ?? writeMultiplier) + : writeMultiplier; + txn.rateDetail = { input: inputMultiplier, write: writeMultiplier, read: readMultiplier, ...(write1hAbs > 0 && { write1h: write1hMultiplier }), + ...(write12hAbs > 0 && { write12h: write12hMultiplier }), }; const totalPromptTokens = inputAbs + writeAbs + readAbs; const promptValue = inputAbs * Math.abs(inputMultiplier) + - (writeAbs - write1hAbs) * Math.abs(writeMultiplier) + + (writeAbs - write1hAbs - write12hAbs) * Math.abs(writeMultiplier) + write1hAbs * Math.abs(write1hMultiplier) + + write12hAbs * Math.abs(write12hMultiplier) + readAbs * Math.abs(readMultiplier); txn.rate = @@ -231,8 +249,9 @@ export function createTransactionMethods( txn.tokenValue = -( inputAbs * inputMultiplier + - (writeAbs - write1hAbs) * writeMultiplier + + (writeAbs - write1hAbs - write12hAbs) * writeMultiplier + write1hAbs * write1hMultiplier + + write12hAbs * write12hMultiplier + readAbs * readMultiplier ); @@ -704,6 +723,7 @@ export function createTransactionMethods( transaction.endpointTokenConfig = txData.endpointTokenConfig; transaction.inputTokenCount = txData.inputTokenCount; transaction.write1hTokens = txData.write1hTokens; + transaction.write12hTokens = txData.write12hTokens; calculateStructuredTokenValue(transaction); diff --git a/packages/data-schemas/src/methods/tx.spec.ts b/packages/data-schemas/src/methods/tx.spec.ts index ea3cb1e549..62545b9cae 100644 --- a/packages/data-schemas/src/methods/tx.spec.ts +++ b/packages/data-schemas/src/methods/tx.spec.ts @@ -3087,9 +3087,11 @@ describe('Long-Context Premium Cache Pricing', () => { it('should scale cache write/read by the same long-context multiplier as input', () => { for (const model of premiumCacheModels) { const inputRatio = premiumTokenValues[model].prompt / tokenValues[model].prompt; - expect(premiumCacheTokenValues[model].write).toBeCloseTo( - cacheTokenValues[model].write * inputRatio, - ); + if (premiumCacheTokenValues[model].write != null) { + expect(premiumCacheTokenValues[model].write).toBeCloseTo( + cacheTokenValues[model].write * inputRatio, + ); + } expect(premiumCacheTokenValues[model].read).toBeCloseTo( cacheTokenValues[model].read * inputRatio, ); @@ -3101,7 +3103,7 @@ describe('Long-Context Premium Cache Pricing', () => { const { threshold } = premiumCacheTokenValues[model]; for (const cacheType of ['write', 'read'] as const) { expect(getCacheMultiplier({ model, cacheType, inputTokenCount: threshold + 1 })).toBe( - premiumCacheTokenValues[model][cacheType], + premiumCacheTokenValues[model][cacheType] ?? null, ); expect(getCacheMultiplier({ model, cacheType, inputTokenCount: threshold })).toBe( cacheTokenValues[model][cacheType], diff --git a/packages/data-schemas/src/methods/tx.ts b/packages/data-schemas/src/methods/tx.ts index 626eb4fc13..1c6a490509 100644 --- a/packages/data-schemas/src/methods/tx.ts +++ b/packages/data-schemas/src/methods/tx.ts @@ -167,6 +167,7 @@ export const tokenValues: Record 'gpt-5.6': { prompt: 4, completion: 20 }, 'gpt-5.6-terra': { prompt: 2, completion: 12 }, 'gpt-5.6-luna': { prompt: 0.2, completion: 1.2 }, + 'gpt-6-cyber': { prompt: 20, completion: 100 }, 'gpt-6-astra': { prompt: 10, completion: 50 }, 'gpt-6-sol': { prompt: 2, completion: 10 }, 'gpt-6.1-terra': { prompt: 1, completion: 5 }, @@ -385,7 +386,9 @@ export const tokenValues: Record * Mapping of model token sizes to their respective multipliers for cached input, read and write. * The rates are 1 USD per 1M tokens. */ -export const cacheTokenValues: Record = { +type CacheRates = { write: number; read: number; write12h?: number }; + +export const cacheTokenValues: Record = { 'claude-3.7-sonnet': { write: 3.75, read: 0.3 }, 'claude-3-7-sonnet': { write: 3.75, read: 0.3 }, 'claude-3.5-sonnet': { write: 3.75, read: 0.3 }, @@ -433,6 +436,7 @@ export const cacheTokenValues: Record = 'gpt-5.6': { write: 5, read: 0.4 }, 'gpt-5.6-terra': { write: 2.5, read: 0.2 }, 'gpt-5.6-luna': { write: 0.25, read: 0.02 }, + 'gpt-6-cyber': { write: 25, write12h: 50, read: 2 }, 'gpt-6-astra': { write: 12.5, read: 1 }, 'gpt-6-sol': { write: 2.5, read: 0.2 }, 'gpt-6.1-terra': { write: 1.25, read: 0.05 }, @@ -499,6 +503,7 @@ export const premiumTokenValues: Record< 'gpt-5.6': { threshold: 272000, prompt: 8, completion: 30 }, 'gpt-5.6-terra': { threshold: 272000, prompt: 4, completion: 18 }, 'gpt-5.6-luna': { threshold: 272000, prompt: 0.4, completion: 1.8 }, + 'gpt-6-cyber': { threshold: 272000, prompt: 40, completion: 150 }, 'gpt-6-astra': { threshold: 272000, prompt: 20, completion: 75 }, 'gpt-6-sol': { threshold: 272000, prompt: 4, completion: 15 }, 'gpt-6.1-terra': { threshold: 272000, prompt: 2, completion: 7.5 }, @@ -520,7 +525,7 @@ export const premiumTokenValues: Record< */ export const premiumCacheTokenValues: Record< string, - { threshold: number; write: number; read: number } + { threshold: number; write?: number; read: number } > = { 'claude-haiku-5-5': { threshold: 100000, write: 0.625, read: 0.05 }, 'claude-haiku-5.5': { threshold: 100000, write: 0.625, read: 0.05 }, @@ -529,6 +534,8 @@ export const premiumCacheTokenValues: Record< 'gpt-5.6': { threshold: 272000, write: 10, read: 0.8 }, 'gpt-5.6-terra': { threshold: 272000, write: 5, read: 0.4 }, 'gpt-5.6-luna': { threshold: 272000, write: 0.5, read: 0.04 }, + // OpenAI publishes no long-context cache-write rate for Cyber. + 'gpt-6-cyber': { threshold: 272000, read: 4 }, 'gpt-6-astra': { threshold: 272000, write: 25, read: 2 }, 'gpt-6-sol': { threshold: 272000, write: 5, read: 0.4 }, 'gpt-6.1-terra': { threshold: 272000, write: 2.5, read: 0.1 }, @@ -538,7 +545,7 @@ export const premiumCacheTokenValues: Record< }; /** `write1h` prices Anthropic's 1-hour prompt cache writes; `write` is the 5-minute rate */ -type CacheType = 'write' | 'read' | 'write1h'; +type CacheType = 'write' | 'read' | 'write1h' | 'write12h'; /** Anthropic bills 1-hour cache writes at 2x the input rate on every model and tier */ export const ONE_HOUR_CACHE_WRITE_FACTOR = 2; @@ -726,13 +733,13 @@ export function createTxMethods( valueKey: string, cacheType: 'write' | 'read', inputTokenCount?: number | null, - ): number | null { + ): number | null | undefined { if (inputTokenCount == null) { - return null; + return undefined; } const premiumEntry = premiumCacheTokenValues[valueKey]; if (!premiumEntry || inputTokenCount <= premiumEntry.threshold) { - return null; + return undefined; } return premiumEntry[cacheType] ?? null; } @@ -759,11 +766,27 @@ export function createTxMethods( endpointTokenConfig?: Record>; inputTokenCount?: number | null; }): number | null { + if (cacheType === 'write12h') { + const override = model ? endpointTokenConfig?.[model] : undefined; + if (override != null) return override.write12h ?? null; + const key = valueKey ?? (model ? getValueKey(model, endpoint) : undefined); + if ( + !key || + (inputTokenCount != null && + inputTokenCount > (premiumCacheTokenValues[key]?.threshold ?? Infinity)) + ) + return null; + return cacheTokenValues[key]?.write12h ?? null; + } + if (cacheType === 'write1h') { const override = model ? endpointTokenConfig?.[model]?.write1h : undefined; if (override != null) { return override; } + if ((valueKey ?? (model ? getValueKey(model, endpoint) : undefined)) === 'gpt-6-cyber') { + return null; + } const promptRate = getMultiplier({ model, valueKey, @@ -785,11 +808,8 @@ export function createTxMethods( } if (valueKey && cacheType) { - return ( - getPremiumCacheRate(valueKey, cacheType, inputTokenCount) ?? - cacheTokenValues[valueKey]?.[cacheType] ?? - null - ); + const premium = getPremiumCacheRate(valueKey, cacheType, inputTokenCount); + return premium !== undefined ? premium : (cacheTokenValues[valueKey]?.[cacheType] ?? null); } if (!cacheType || !model) { @@ -801,11 +821,8 @@ export function createTxMethods( return null; } - return ( - getPremiumCacheRate(valueKey, cacheType, inputTokenCount) ?? - cacheTokenValues[valueKey]?.[cacheType] ?? - null - ); + const premium = getPremiumCacheRate(valueKey, cacheType, inputTokenCount); + return premium !== undefined ? premium : (cacheTokenValues[valueKey]?.[cacheType] ?? null); } return {