mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-06-28 02:11:30 +00:00
* refactor(Chains/llms): allow passing callbacks * refactor(BaseClient): accurately count completion tokens as generation only * refactor(OpenAIClient): remove unused getTokenCountForResponse, pass streaming var and callbacks in initializeLLM * wip: summary prompt tokens * refactor(summarizeMessages): new cut-off strategy that generates a better summary by adding context from beginning, truncating the middle, and providing the end wip: draft out relevant providers and variables for token tracing * refactor(createLLM): make streaming prop false by default * chore: remove use of getTokenCountForResponse * refactor(agents): use BufferMemory as ConversationSummaryBufferMemory token usage not easy to trace * chore: remove passing of streaming prop, also console log useful vars for tracing * feat: formatFromLangChain helper function to count tokens for ChatModelStart * refactor(initializeLLM): add role for LLM tracing * chore(formatFromLangChain): update JSDoc * feat(formatMessages): formats langChain messages into OpenAI payload format * chore: install openai-chat-tokens * refactor(formatMessage): optimize conditional langChain logic fix(formatFromLangChain): fix destructuring * feat: accurate prompt tokens for ChatModelStart before generation * refactor(handleChatModelStart): move to callbacks dir, use factory function * refactor(initializeLLM): rename 'role' to 'context' * feat(Balance/Transaction): new schema/models for tracking token spend refactor(Key): factor out model export to separate file * refactor(initializeClient): add req,res objects to client options * feat: add-balance script to add to an existing users' token balance refactor(Transaction): use multiplier map/function, return balance update * refactor(Tx): update enum for tokenType, return 1 for multiplier if no map match * refactor(Tx): add fair fallback value multiplier incase the config result is undefined * refactor(Balance): rename 'tokens' to 'tokenCredits' * feat: balance check, add tx.js for new tx-related methods and tests * chore(summaryPrompts): update prompt token count * refactor(callbacks): pass req, res wip: check balance * refactor(Tx): make convoId a String type, fix(calculateTokenValue) * refactor(BaseClient): add conversationId as client prop when assigned * feat(RunManager): track LLM runs with manager, track token spend from LLM, refactor(OpenAIClient): use RunManager to create callbacks, pass user prop to langchain api calls * feat(spendTokens): helper to spend prompt/completion tokens * feat(checkBalance): add helper to check, log, deny request if balance doesn't have enough funds refactor(Balance): static check method to return object instead of boolean now wip(OpenAIClient): implement use of checkBalance * refactor(initializeLLM): add token buffer to assure summary isn't generated when subsequent payload is too large refactor(OpenAIClient): add checkBalance refactor(createStartHandler): add checkBalance * chore: remove prompt and completion token logging from route handler * chore(spendTokens): add JSDoc * feat(logTokenCost): record transactions for basic api calls * chore(ask/edit): invoke getResponseSender only once per API call * refactor(ask/edit): pass promptTokens to getIds and include in abort data * refactor(getIds -> getReqData): rename function * refactor(Tx): increase value if incomplete message * feat: record tokenUsage when message is aborted * refactor: subtract tokens when payload includes function_call * refactor: add namespace for token_balance * fix(spendTokens): only execute if corresponding token type amounts are defined * refactor(checkBalance): throws Error if not enough token credits * refactor(runTitleChain): pass and use signal, spread object props in create helpers, and use 'call' instead of 'run' * fix(abortMiddleware): circular dependency, and default to empty string for completionTokens * fix: properly cancel title requests when there isn't enough tokens to generate * feat(predictNewSummary): custom chain for summaries to allow signal passing refactor(summaryBuffer): use new custom chain * feat(RunManager): add getRunByConversationId method, refactor: remove run and throw llm error on handleLLMError * refactor(createStartHandler): if summary, add error details to runs * fix(OpenAIClient): support aborting from summarization & showing error to user refactor(summarizeMessages): remove unnecessary operations counting summaryPromptTokens and note for alternative, pass signal to summaryBuffer * refactor(logTokenCost -> recordTokenUsage): rename * refactor(checkBalance): include promptTokens in errorMessage * refactor(checkBalance/spendTokens): move to models dir * fix(createLanguageChain): correctly pass config * refactor(initializeLLM/title): add tokenBuffer of 150 for balance check * refactor(openAPIPlugin): pass signal and memory, filter functions by the one being called * refactor(createStartHandler): add error to run if context is plugins as well * refactor(RunManager/handleLLMError): throw error immediately if plugins, don't remove run * refactor(PluginsClient): pass memory and signal to tools, cleanup error handling logic * chore: use absolute equality for addTitle condition * refactor(checkBalance): move checkBalance to execute after userMessage and tokenCounts are saved, also make conditional * style: icon changes to match official * fix(BaseClient): getTokenCountForResponse -> getTokenCount * fix(formatLangChainMessages): add kwargs as fallback prop from lc_kwargs, update JSDoc * refactor(Tx.create): does not update balance if CHECK_BALANCE is not enabled * fix(e2e/cleanUp): cleanup new collections, import all model methods from index * fix(config/add-balance): add uncaughtException listener * fix: circular dependency * refactor(initializeLLM/checkBalance): append new generations to errorMessage if cost exceeds balance * fix(handleResponseMessage): only record token usage in this method if not error and completion is not skipped * fix(createStartHandler): correct condition for generations * chore: bump postcss due to moderate severity vulnerability * chore: bump zod due to low severity vulnerability * chore: bump openai & data-provider version * feat(types): OpenAI Message types * chore: update bun lockfile * refactor(CodeBlock): add error block formatting * refactor(utils/Plugin): factor out formatJSON and cn to separate files (json.ts and cn.ts), add extractJSON * chore(logViolation): delete user_id after error is logged * refactor(getMessageError -> Error): change to React.FC, add token_balance handling, use extractJSON to determine JSON instead of regex * fix(DALL-E): use latest openai SDK * chore: reorganize imports, fix type issue * feat(server): add balance route * fix(api/models): add auth * feat(data-provider): /api/balance query * feat: show balance if checking is enabled, refetch on final message or error * chore: update docs, .env.example with token_usage info, add balance script command * fix(Balance): fallback to empty obj for balance query * style: slight adjustment of balance element * docs(token_usage): add PR notes
465 lines
12 KiB
TypeScript
465 lines
12 KiB
TypeScript
import {
|
|
UseQueryOptions,
|
|
useQuery,
|
|
useMutation,
|
|
useQueryClient,
|
|
UseMutationResult,
|
|
QueryObserverResult,
|
|
} from '@tanstack/react-query';
|
|
import * as t from './types';
|
|
import * as s from './schemas';
|
|
import * as dataService from './data-service';
|
|
|
|
export enum QueryKeys {
|
|
messages = 'messsages',
|
|
allConversations = 'allConversations',
|
|
conversation = 'conversation',
|
|
searchEnabled = 'searchEnabled',
|
|
user = 'user',
|
|
name = 'name', // user key name
|
|
models = 'models',
|
|
balance = 'balance',
|
|
endpoints = 'endpoints',
|
|
presets = 'presets',
|
|
searchResults = 'searchResults',
|
|
tokenCount = 'tokenCount',
|
|
availablePlugins = 'availablePlugins',
|
|
startupConfig = 'startupConfig',
|
|
}
|
|
|
|
export const useAbortRequestWithMessage = (): UseMutationResult<
|
|
void,
|
|
Error,
|
|
{ endpoint: string; abortKey: string; message: string }
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(
|
|
({ endpoint, abortKey, message }) =>
|
|
dataService.abortRequestWithMessage(endpoint, abortKey, message),
|
|
{
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.balance]);
|
|
},
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useGetUserQuery = (
|
|
config?: UseQueryOptions<t.TUser>,
|
|
): QueryObserverResult<t.TUser> => {
|
|
return useQuery<t.TUser>([QueryKeys.user], () => dataService.getUser(), {
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
retry: false,
|
|
...config,
|
|
});
|
|
};
|
|
|
|
export const useGetMessagesByConvoId = (
|
|
id: string,
|
|
config?: UseQueryOptions<s.TMessage[]>,
|
|
): QueryObserverResult<s.TMessage[]> => {
|
|
return useQuery<s.TMessage[]>(
|
|
[QueryKeys.messages, id],
|
|
() => dataService.getMessagesByConvoId(id),
|
|
{
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
...config,
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useGetUserBalance = (
|
|
config?: UseQueryOptions<string>,
|
|
): QueryObserverResult<string> => {
|
|
return useQuery<string>([QueryKeys.balance], () => dataService.getUserBalance(), {
|
|
refetchOnWindowFocus: true,
|
|
refetchOnReconnect: true,
|
|
refetchOnMount: true,
|
|
...config,
|
|
});
|
|
};
|
|
|
|
export const useGetConversationByIdQuery = (
|
|
id: string,
|
|
config?: UseQueryOptions<s.TConversation>,
|
|
): QueryObserverResult<s.TConversation> => {
|
|
return useQuery<s.TConversation>(
|
|
[QueryKeys.conversation, id],
|
|
() => dataService.getConversationById(id),
|
|
{
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
...config,
|
|
},
|
|
);
|
|
};
|
|
|
|
//This isn't ideal because its just a query and we're using mutation, but it was the only way
|
|
//to make it work with how the Chat component is structured
|
|
export const useGetConversationByIdMutation = (id: string): UseMutationResult<s.TConversation> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(() => dataService.getConversationById(id), {
|
|
// onSuccess: (res: s.TConversation) => {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.conversation, id]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useUpdateConversationMutation = (
|
|
id: string,
|
|
): UseMutationResult<
|
|
t.TUpdateConversationResponse,
|
|
unknown,
|
|
t.TUpdateConversationRequest,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(
|
|
(payload: t.TUpdateConversationRequest) => dataService.updateConversation(payload),
|
|
{
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.conversation, id]);
|
|
queryClient.invalidateQueries([QueryKeys.allConversations]);
|
|
},
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useUpdateMessageMutation = (
|
|
id: string,
|
|
): UseMutationResult<unknown, unknown, t.TUpdateMessageRequest, unknown> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: t.TUpdateMessageRequest) => dataService.updateMessage(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.messages, id]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useUpdateUserKeysMutation = (): UseMutationResult<
|
|
t.TUser,
|
|
unknown,
|
|
t.TUpdateUserKeyRequest,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: t.TUpdateUserKeyRequest) => dataService.updateUserKey(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.name]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useDeleteConversationMutation = (
|
|
id?: string,
|
|
): UseMutationResult<
|
|
t.TDeleteConversationResponse,
|
|
unknown,
|
|
t.TDeleteConversationRequest,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(
|
|
(payload: t.TDeleteConversationRequest) => dataService.deleteConversation(payload),
|
|
{
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.conversation, id]);
|
|
queryClient.invalidateQueries([QueryKeys.allConversations]);
|
|
},
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useClearConversationsMutation = (): UseMutationResult<unknown> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(() => dataService.clearAllConversations(), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.allConversations]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useRevokeUserKeyMutation = (name: string): UseMutationResult<unknown> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(() => dataService.revokeUserKey(name), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.name]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useRevokeAllUserKeysMutation = (): UseMutationResult<unknown> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(() => dataService.revokeAllUserKeys(), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.name]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useGetConversationsQuery = (
|
|
pageNumber: string,
|
|
config?: UseQueryOptions<t.TGetConversationsResponse>,
|
|
): QueryObserverResult<t.TGetConversationsResponse> => {
|
|
return useQuery<t.TGetConversationsResponse>(
|
|
[QueryKeys.allConversations, pageNumber],
|
|
() => dataService.getConversations(pageNumber),
|
|
{
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
retry: 1,
|
|
...config,
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useGetSearchEnabledQuery = (
|
|
config?: UseQueryOptions<boolean>,
|
|
): QueryObserverResult<boolean> => {
|
|
return useQuery<boolean>([QueryKeys.searchEnabled], () => dataService.getSearchEnabled(), {
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
...config,
|
|
});
|
|
};
|
|
|
|
export const useGetEndpointsQuery = (): QueryObserverResult<t.TEndpointsConfig> => {
|
|
return useQuery([QueryKeys.endpoints], () => dataService.getAIEndpoints(), {
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
});
|
|
};
|
|
|
|
export const useGetModelsQuery = (): QueryObserverResult<t.TModelsConfig> => {
|
|
return useQuery([QueryKeys.models], () => dataService.getModels(), {
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
});
|
|
};
|
|
|
|
export const useCreatePresetMutation = (): UseMutationResult<
|
|
s.TPreset[],
|
|
unknown,
|
|
s.TPreset,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: s.TPreset) => dataService.createPreset(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.presets]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useUpdatePresetMutation = (): UseMutationResult<
|
|
s.TPreset[],
|
|
unknown,
|
|
s.TPreset,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: s.TPreset) => dataService.updatePreset(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.presets]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useGetPresetsQuery = (
|
|
config?: UseQueryOptions<s.TPreset[]>,
|
|
): QueryObserverResult<s.TPreset[], unknown> => {
|
|
return useQuery<s.TPreset[]>([QueryKeys.presets], () => dataService.getPresets(), {
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
...config,
|
|
});
|
|
};
|
|
|
|
export const useDeletePresetMutation = (): UseMutationResult<
|
|
s.TPreset[],
|
|
unknown,
|
|
s.TPreset | object,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: s.TPreset | object) => dataService.deletePreset(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.presets]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useSearchQuery = (
|
|
searchQuery: string,
|
|
pageNumber: string,
|
|
config?: UseQueryOptions<t.TSearchResults>,
|
|
): QueryObserverResult<t.TSearchResults> => {
|
|
return useQuery<t.TSearchResults>(
|
|
[QueryKeys.searchResults, pageNumber, searchQuery],
|
|
() => dataService.searchConversations(searchQuery, pageNumber),
|
|
{
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
...config,
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useUpdateTokenCountMutation = (): UseMutationResult<
|
|
t.TUpdateTokenCountResponse,
|
|
unknown,
|
|
{ text: string },
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(({ text }: { text: string }) => dataService.updateTokenCount(text), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.tokenCount]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useLoginUserMutation = (): UseMutationResult<
|
|
t.TLoginResponse,
|
|
unknown,
|
|
t.TLoginUser,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: t.TLoginUser) => dataService.login(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.user]);
|
|
},
|
|
onMutate: () => {
|
|
queryClient.invalidateQueries([QueryKeys.models]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useRegisterUserMutation = (): UseMutationResult<
|
|
unknown,
|
|
unknown,
|
|
t.TRegisterUser,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: t.TRegisterUser) => dataService.register(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.user]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useLogoutUserMutation = (): UseMutationResult<unknown> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(() => dataService.logout(), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.user]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useRefreshTokenMutation = (): UseMutationResult<
|
|
t.TRefreshTokenResponse,
|
|
unknown,
|
|
unknown,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation(() => dataService.refreshToken(), {
|
|
onMutate: () => {
|
|
queryClient.invalidateQueries([QueryKeys.models]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useUserKeyQuery = (
|
|
name: string,
|
|
config?: UseQueryOptions<t.TCheckUserKeyResponse>,
|
|
): QueryObserverResult<t.TCheckUserKeyResponse> => {
|
|
return useQuery<t.TCheckUserKeyResponse>(
|
|
[QueryKeys.name, name],
|
|
() => {
|
|
if (!name) {
|
|
return Promise.resolve({ expiresAt: '' });
|
|
}
|
|
return dataService.userKeyQuery(name);
|
|
},
|
|
{
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
retry: false,
|
|
...config,
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useRequestPasswordResetMutation = (): UseMutationResult<
|
|
t.TRequestPasswordResetResponse,
|
|
unknown,
|
|
t.TRequestPasswordReset,
|
|
unknown
|
|
> => {
|
|
return useMutation((payload: t.TRequestPasswordReset) =>
|
|
dataService.requestPasswordReset(payload),
|
|
);
|
|
};
|
|
|
|
export const useResetPasswordMutation = (): UseMutationResult<
|
|
unknown,
|
|
unknown,
|
|
t.TResetPassword,
|
|
unknown
|
|
> => {
|
|
return useMutation((payload: t.TResetPassword) => dataService.resetPassword(payload));
|
|
};
|
|
|
|
export const useAvailablePluginsQuery = (): QueryObserverResult<s.TPlugin[]> => {
|
|
return useQuery<s.TPlugin[]>(
|
|
[QueryKeys.availablePlugins],
|
|
() => dataService.getAvailablePlugins(),
|
|
{
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
},
|
|
);
|
|
};
|
|
|
|
export const useUpdateUserPluginsMutation = (): UseMutationResult<
|
|
t.TUser,
|
|
unknown,
|
|
t.TUpdateUserPlugins,
|
|
unknown
|
|
> => {
|
|
const queryClient = useQueryClient();
|
|
return useMutation((payload: t.TUpdateUserPlugins) => dataService.updateUserPlugins(payload), {
|
|
onSuccess: () => {
|
|
queryClient.invalidateQueries([QueryKeys.user]);
|
|
},
|
|
});
|
|
};
|
|
|
|
export const useGetStartupConfig = (): QueryObserverResult<t.TStartupConfig> => {
|
|
return useQuery<t.TStartupConfig>(
|
|
[QueryKeys.startupConfig],
|
|
() => dataService.getStartupConfig(),
|
|
{
|
|
refetchOnWindowFocus: false,
|
|
refetchOnReconnect: false,
|
|
refetchOnMount: false,
|
|
},
|
|
);
|
|
};
|