From d7ef4590eabe6d392e23e007460c955ddbd32cf9 Mon Sep 17 00:00:00 2001 From: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Sun, 26 Nov 2023 14:44:57 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Fix:=20Resolve=20Anthropic=20Cli?= =?UTF-8?q?ent=20Issues=20=20=F0=9F=A7=A0=20(#1226)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: correct preset title for Anthropic endpoint * fix(Settings/Anthropic): show correct default value for LLM temperature * fix(AnthropicClient): use `getModelMaxTokens` to get the correct LLM max context tokens, correctly set default temperature to 1, use only 2 params for class constructor, use `getResponseSender` to add correct sender to response message * refactor(/api/ask|edit/anthropic): save messages to database after the final response is sent to the client, and do not save conversation from route controller * fix(initializeClient/anthropic): correctly pass client options (endpointOption) to class initialization * feat(ModelService/Anthropic): add claude-1.2 --- api/app/clients/AnthropicClient.js | 44 ++++++++++--------- api/server/routes/ask/anthropic.js | 17 +++---- api/server/routes/edit/anthropic.js | 10 +++-- .../endpoints/anthropic/initializeClient.js | 21 +++++---- api/server/services/ModelService.js | 1 + api/utils/tokens.js | 2 + api/utils/tokens.spec.js | 19 ++++++++ .../Endpoints/Settings/Anthropic.tsx | 2 +- client/src/utils/presets.ts | 2 +- 9 files changed, 73 insertions(+), 45 deletions(-) diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index cdc105d258..2d124c96da 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -1,7 +1,8 @@ -// const { Agent, ProxyAgent } = require('undici'); -const BaseClient = require('./BaseClient'); -const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); const Anthropic = require('@anthropic-ai/sdk'); +const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); +const { getResponseSender, EModelEndpoint } = require('~/server/routes/endpoints/schemas'); +const { getModelMaxTokens } = require('~/utils'); +const BaseClient = require('./BaseClient'); const HUMAN_PROMPT = '\n\nHuman:'; const AI_PROMPT = '\n\nAssistant:'; @@ -9,13 +10,9 @@ const AI_PROMPT = '\n\nAssistant:'; const tokenizersCache = {}; class AnthropicClient extends BaseClient { - constructor(apiKey, options = {}, cacheOptions = {}, baseURL) { - super(apiKey, options, cacheOptions); + constructor(apiKey, options = {}) { + super(apiKey, options); this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY; - this.sender = 'Anthropic'; - if (baseURL) { - this.baseURL = baseURL; - } this.userLabel = HUMAN_PROMPT; this.assistantLabel = AI_PROMPT; this.setOptions(options); @@ -43,13 +40,13 @@ class AnthropicClient extends BaseClient { ...modelOptions, // set some good defaults (check for undefined in some cases because they may be 0) model: modelOptions.model || 'claude-1', - temperature: typeof modelOptions.temperature === 'undefined' ? 0.7 : modelOptions.temperature, // 0 - 1, 0.7 is recommended + temperature: typeof modelOptions.temperature === 'undefined' ? 1 : modelOptions.temperature, // 0 - 1, 1 is default topP: typeof modelOptions.topP === 'undefined' ? 0.7 : modelOptions.topP, // 0 - 1, default: 0.7 topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40 stop: modelOptions.stop, // no stop method for now }; - this.maxContextTokens = this.options.maxContextTokens || 99999; + this.maxContextTokens = getModelMaxTokens(this.modelOptions.model) ?? 100000; this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500; this.maxPromptTokens = this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens; @@ -62,6 +59,14 @@ class AnthropicClient extends BaseClient { ); } + this.sender = + this.options.sender ?? + getResponseSender({ + model: this.modelOptions.model, + endpoint: EModelEndpoint.anthropic, + modelLabel: this.options.modelLabel, + }); + this.startToken = '||>'; this.endToken = ''; this.gptEncoder = this.constructor.getTokenizer('cl100k_base'); @@ -81,16 +86,15 @@ class AnthropicClient extends BaseClient { } getClient() { - if (this.baseURL) { - return new Anthropic({ - apiKey: this.apiKey, - baseURL: this.baseURL, - }); - } else { - return new Anthropic({ - apiKey: this.apiKey, - }); + const options = { + apiKey: this.apiKey, + }; + + if (this.options.reverseProxyUrl) { + options.baseURL = this.options.reverseProxyUrl; } + + return new Anthropic(options); } async buildMessages(messages, parentMessageId) { diff --git a/api/server/routes/ask/anthropic.js b/api/server/routes/ask/anthropic.js index 5d4725e86c..233d3dc259 100644 --- a/api/server/routes/ask/anthropic.js +++ b/api/server/routes/ask/anthropic.js @@ -9,9 +9,9 @@ const { setHeaders, validateEndpoint, buildEndpointOption, -} = require('../../middleware'); -const { saveMessage, getConvoTitle, saveConvo, getConvo } = require('../../../models'); -const { sendMessage, createOnProgress } = require('../../utils'); +} = require('~/server/middleware'); +const { saveMessage, getConvoTitle, getConvo } = require('~/models'); +const { sendMessage, createOnProgress } = require('~/server/utils'); router.post('/abort', handleAbort()); @@ -109,14 +109,6 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, response.parentMessageId = overrideParentMessageId; } - await saveConvo(user, { - ...endpointOption, - ...endpointOption.modelOptions, - conversationId, - endpoint: 'anthropic', - }); - - await saveMessage({ ...response, user }); sendMessage(res, { title: await getConvoTitle(user, conversationId), final: true, @@ -126,6 +118,9 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, }); res.end(); + await saveMessage({ ...response, user }); + await saveMessage(userMessage); + // TODO: add anthropic titling } catch (error) { const partialText = getPartialText(); diff --git a/api/server/routes/edit/anthropic.js b/api/server/routes/edit/anthropic.js index 185d714ef0..66a8a64395 100644 --- a/api/server/routes/edit/anthropic.js +++ b/api/server/routes/edit/anthropic.js @@ -9,9 +9,9 @@ const { setHeaders, validateEndpoint, buildEndpointOption, -} = require('../../middleware'); -const { saveMessage, getConvoTitle, getConvo } = require('../../../models'); -const { sendMessage, createOnProgress } = require('../../utils'); +} = require('~/server/middleware'); +const { saveMessage, getConvoTitle, getConvo } = require('~/models'); +const { sendMessage, createOnProgress } = require('~/server/utils'); router.post('/abort', handleAbort()); @@ -119,7 +119,6 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, response.parentMessageId = overrideParentMessageId; } - await saveMessage({ ...response, user }); sendMessage(res, { title: await getConvoTitle(user, conversationId), final: true, @@ -129,6 +128,9 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, }); res.end(); + await saveMessage({ ...response, user }); + await saveMessage(userMessage); + // TODO: add anthropic titling } catch (error) { const partialText = getPartialText(); diff --git a/api/server/routes/endpoints/anthropic/initializeClient.js b/api/server/routes/endpoints/anthropic/initializeClient.js index 5160611e7e..4700da6c84 100644 --- a/api/server/routes/endpoints/anthropic/initializeClient.js +++ b/api/server/routes/endpoints/anthropic/initializeClient.js @@ -1,14 +1,14 @@ -const { AnthropicClient } = require('../../../../app'); -const { getUserKey, checkUserKeyExpiry } = require('../../../services/UserService'); +const { AnthropicClient } = require('~/app'); +const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); -const initializeClient = async ({ req, res }) => { - const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY; +const initializeClient = async ({ req, res, endpointOption }) => { + const { ANTHROPIC_API_KEY, ANTHROPIC_REVERSE_PROXY } = process.env; const expiresAt = req.body.key; const isUserProvided = ANTHROPIC_API_KEY === 'user_provided'; - let anthropicApiKey = isUserProvided ? await getAnthropicUserKey(req.user.id) : ANTHROPIC_API_KEY; - let reverseProxy = process.env.ANTHROPIC_REVERSE_PROXY || undefined; - console.log('ANTHROPIC_REVERSE_PROXY', reverseProxy); + const anthropicApiKey = isUserProvided + ? await getAnthropicUserKey(req.user.id) + : ANTHROPIC_API_KEY; if (expiresAt && isUserProvided) { checkUserKeyExpiry( @@ -17,7 +17,12 @@ const initializeClient = async ({ req, res }) => { ); } - const client = new AnthropicClient(anthropicApiKey, { req, res }, {}, reverseProxy); + const client = new AnthropicClient(anthropicApiKey, { + req, + res, + reverseProxyUrl: ANTHROPIC_REVERSE_PROXY ?? null, + ...endpointOption, + }); return { client, diff --git a/api/server/services/ModelService.js b/api/server/services/ModelService.js index 086e8b7396..d103bd1c17 100644 --- a/api/server/services/ModelService.js +++ b/api/server/services/ModelService.js @@ -117,6 +117,7 @@ const getAnthropicModels = () => { let models = [ 'claude-2.1', 'claude-2', + 'claude-1.2', 'claude-1', 'claude-1-100k', 'claude-instant-1', diff --git a/api/utils/tokens.js b/api/utils/tokens.js index b422acee46..c956d53ce5 100644 --- a/api/utils/tokens.js +++ b/api/utils/tokens.js @@ -51,6 +51,8 @@ const maxTokensMap = { 'gpt-3.5-turbo-16k-0613': 15999, 'gpt-3.5-turbo-1106': 16380, // -5 from max 'gpt-4-1106': 127995, // -5 from max + 'claude-2.1': 200000, + 'claude-': 100000, }; /** diff --git a/api/utils/tokens.spec.js b/api/utils/tokens.spec.js index 5cf488c2ca..8b5150770f 100644 --- a/api/utils/tokens.spec.js +++ b/api/utils/tokens.spec.js @@ -62,6 +62,25 @@ describe('getModelMaxTokens', () => { expect(getModelMaxTokens('gpt-4-1106-preview')).toBe(maxTokensMap['gpt-4-1106']); expect(getModelMaxTokens('gpt-4-1106-vision-preview')).toBe(maxTokensMap['gpt-4-1106']); }); + + test('should return correct tokens for Anthropic models', () => { + const models = [ + 'claude-2.1', + 'claude-2', + 'claude-1.2', + 'claude-1', + 'claude-1-100k', + 'claude-instant-1', + 'claude-instant-1-100k', + ]; + + const claude21MaxTokens = maxTokensMap['claude-2.1']; + const claudeMaxTokens = maxTokensMap['claude-']; + models.forEach((model) => { + const expectedTokens = model === 'claude-2.1' ? claude21MaxTokens : claudeMaxTokens; + expect(getModelMaxTokens(model)).toEqual(expectedTokens); + }); + }); }); describe('matchModelName', () => { diff --git a/client/src/components/Endpoints/Settings/Anthropic.tsx b/client/src/components/Endpoints/Settings/Anthropic.tsx index d510b28e02..617cddc01d 100644 --- a/client/src/components/Endpoints/Settings/Anthropic.tsx +++ b/client/src/components/Endpoints/Settings/Anthropic.tsx @@ -86,7 +86,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
{ if (model) { _title += `: ${model}`; } - } else if (endpoint === EModelEndpoint.google) { + } else if (endpoint === EModelEndpoint.google || endpoint === EModelEndpoint.anthropic) { if (modelLabel) { _title = modelLabel; }