mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-07-10 08:13:46 +00:00
refactor: share string normalization helper
This commit is contained in:
parent
a2791bee5d
commit
be4d874cef
6 changed files with 19 additions and 8 deletions
|
|
@ -1,7 +1,8 @@
|
|||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { RunConfig } from '@librechat/agents';
|
||||
import { resolveLangfuseTenantDestination } from './tenantDestinations';
|
||||
import { isTrueEnv, normalizeString } from './utils';
|
||||
import { normalizeString } from '~/utils/text';
|
||||
import { isTrueEnv } from './utils';
|
||||
|
||||
type LangfuseRunConfig = NonNullable<RunConfig['langfuse']>;
|
||||
type LangfuseRunConfigWithTraceAttributes = LangfuseRunConfig & {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import { isFalseEnv, isTrueEnv, normalizeString, toBasicAuthorization } from './utils';
|
||||
import { resolveLangfuseTenantDestination } from './tenantDestinations';
|
||||
import { isFalseEnv, isTrueEnv, toBasicAuthorization } from './utils';
|
||||
import { normalizeString } from '~/utils/text';
|
||||
|
||||
const DEFAULT_BASE_URL = 'https://cloud.langfuse.com';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { normalizeString } from './utils';
|
||||
import { normalizeString } from '~/utils/text';
|
||||
|
||||
const DEFAULT_TENANT_DESTINATIONS: Array<[string, string]> = [
|
||||
['eu', 'https://cloud.langfuse.com'],
|
||||
|
|
|
|||
|
|
@ -1,7 +1,3 @@
|
|||
export function normalizeString(value: unknown): string | undefined {
|
||||
return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
export function isFalseEnv(value?: string): boolean {
|
||||
return value != null && ['0', 'false', 'no', 'off'].includes(value.trim().toLowerCase());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { processTextWithTokenLimit, TokenCountFn } from './text';
|
||||
import { normalizeString, processTextWithTokenLimit, TokenCountFn } from './text';
|
||||
import Tokenizer, { countTokens } from './tokenizer';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
|
|
@ -9,6 +9,15 @@ jest.mock('@librechat/data-schemas', () => ({
|
|||
},
|
||||
}));
|
||||
|
||||
describe('normalizeString', () => {
|
||||
it('trims non-empty strings and treats blank or non-string values as undefined', () => {
|
||||
expect(normalizeString(' value ')).toBe('value');
|
||||
expect(normalizeString(' ')).toBeUndefined();
|
||||
expect(normalizeString(null)).toBeUndefined();
|
||||
expect(normalizeString(123)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* OLD IMPLEMENTATION (Binary Search) - kept for comparison testing
|
||||
* This is the original algorithm that caused CPU spikes
|
||||
|
|
|
|||
|
|
@ -3,6 +3,10 @@ import { logger } from '@librechat/data-schemas';
|
|||
/** Token count function that can be sync or async */
|
||||
export type TokenCountFn = (text: string) => number | Promise<number>;
|
||||
|
||||
export function normalizeString(value: unknown): string | undefined {
|
||||
return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safety buffer multiplier applied to character position estimates during truncation.
|
||||
*
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue