diff --git a/.env.example b/.env.example index a35e685be0..5726655396 100644 --- a/.env.example +++ b/.env.example @@ -138,6 +138,8 @@ NODE_MAX_OLD_SPACE_SIZE=6144 # OTEL_TRACES_SAMPLER=parentbased_always_on # OTEL_LOG_LEVEL=INFO # OTEL_SDK_DISABLED=false +# Enable Redis command-level spans. Disabled by default to keep backend traces high-level. +# OTEL_IOREDIS_TRACING_ENABLED=false #===============================# # Real User Monitoring (Browser) # diff --git a/packages/api/src/telemetry/config.spec.ts b/packages/api/src/telemetry/config.spec.ts index 128f07a5a3..2e87f9cb80 100644 --- a/packages/api/src/telemetry/config.spec.ts +++ b/packages/api/src/telemetry/config.spec.ts @@ -8,6 +8,7 @@ describe('getTelemetryConfig', () => { expect(config.sdkDisabled).toBe(false); expect(config.serviceName).toBe('librechat'); expect(config.healthPath).toBe('/health'); + expect(config.ioredisTracingEnabled).toBe(false); }); it('enables tracing only when OTEL_TRACING_ENABLED is true', () => { @@ -43,4 +44,12 @@ describe('getTelemetryConfig', () => { expect(config.serviceVersion).toBe('0.8.5'); }); + + it('enables ioredis instrumentation only when explicitly configured', () => { + const config = getTelemetryConfig({ + OTEL_IOREDIS_TRACING_ENABLED: 'true', + }); + + expect(config.ioredisTracingEnabled).toBe(true); + }); }); diff --git a/packages/api/src/telemetry/config.ts b/packages/api/src/telemetry/config.ts index 340197aa65..4848bf3be3 100644 --- a/packages/api/src/telemetry/config.ts +++ b/packages/api/src/telemetry/config.ts @@ -6,6 +6,7 @@ export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | ' export interface TelemetryConfig { enabled: boolean; healthPath: string; + ioredisTracingEnabled: boolean; sdkDisabled: boolean; serviceName: string; serviceVersion?: string; @@ -32,10 +33,12 @@ export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): Teleme const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME; const serviceVersion = normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version); + const ioredisTracingEnabled = isTruthy(env.OTEL_IOREDIS_TRACING_ENABLED); return { enabled, serviceName, + ioredisTracingEnabled, sdkDisabled, serviceVersion, healthPath: DEFAULT_HEALTH_PATH, diff --git a/packages/api/src/telemetry/sdk.spec.ts b/packages/api/src/telemetry/sdk.spec.ts index 9e79a49d9c..c361a6cd5c 100644 --- a/packages/api/src/telemetry/sdk.spec.ts +++ b/packages/api/src/telemetry/sdk.spec.ts @@ -1,8 +1,8 @@ import { Socket } from 'node:net'; import { IncomingMessage } from 'node:http'; import { Agent as HttpsAgent } from 'node:https'; -import type { Span } from '@opentelemetry/api'; import type { RequestOptions } from 'node:http'; +import type { Span } from '@opentelemetry/api'; interface HttpInstrumentationOptions { requestHook?: (span: Span, request: object) => void; @@ -191,6 +191,20 @@ describe('telemetry SDK lifecycle', () => { expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1); expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1); expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1); + expect(mockIORedisInstrumentation).not.toHaveBeenCalled(); + expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1); + }); + + it('enables ioredis instrumentation when explicitly configured', () => { + initializeTelemetry({ + OTEL_IOREDIS_TRACING_ENABLED: 'true', + OTEL_TRACING_ENABLED: 'true', + }); + + expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1); + expect(mockHttpInstrumentation).toHaveBeenCalledTimes(1); + expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1); + expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1); expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1); expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1); }); diff --git a/packages/api/src/telemetry/sdk.ts b/packages/api/src/telemetry/sdk.ts index ea5e0a3de2..c3fb90b852 100644 --- a/packages/api/src/telemetry/sdk.ts +++ b/packages/api/src/telemetry/sdk.ts @@ -12,8 +12,8 @@ import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node'; import type { Span, Attributes } from '@opentelemetry/api'; import type { RequestOptions } from 'node:http'; import type { TelemetryConfig, TelemetryStatus } from './config'; -import { getTelemetryConfig } from './config'; import { registerShutdownTask } from '../app/shutdown'; +import { getTelemetryConfig } from './config'; export interface TelemetryController { readonly enabled: boolean; @@ -297,34 +297,39 @@ function getResourceAttributes(config: TelemetryConfig): Attributes { } function createSdk(config: TelemetryConfig): NodeSDK { + const instrumentations: NodeSDKConfiguration['instrumentations'] = [ + new HttpInstrumentation({ + headersToSpanAttributes: { + client: { requestHeaders: [], responseHeaders: [] }, + server: { requestHeaders: [], responseHeaders: [] }, + }, + requestHook: (span: Span, request: object) => { + if (request instanceof IncomingMessage) { + requestSpans.set(request, span); + } + }, + startIncomingSpanHook: (request: IncomingMessage) => + getSanitizedIncomingUrlAttributes(request, config.healthPath), + startOutgoingSpanHook: (request: RequestOptions) => + getSanitizedOutgoingHttpUrlAttributes(request), + ignoreIncomingRequestHook: (request: IncomingMessage) => + shouldIgnoreIncomingRequest(request, config.healthPath), + }), + new ExpressInstrumentation(), + new MongoDBInstrumentation(), + new MongooseInstrumentation(), + new UndiciInstrumentation({ + startSpanHook: (request: UndiciRequestInfo) => getSanitizedUndiciUrlAttributes(request), + }), + ]; + + if (config.ioredisTracingEnabled) { + instrumentations.push(new IORedisInstrumentation()); + } + const sdkConfig: Partial = { resource: resourceFromAttributes(getResourceAttributes(config)), - instrumentations: [ - new HttpInstrumentation({ - headersToSpanAttributes: { - client: { requestHeaders: [], responseHeaders: [] }, - server: { requestHeaders: [], responseHeaders: [] }, - }, - requestHook: (span: Span, request: object) => { - if (request instanceof IncomingMessage) { - requestSpans.set(request, span); - } - }, - startIncomingSpanHook: (request: IncomingMessage) => - getSanitizedIncomingUrlAttributes(request, config.healthPath), - startOutgoingSpanHook: (request: RequestOptions) => - getSanitizedOutgoingHttpUrlAttributes(request), - ignoreIncomingRequestHook: (request: IncomingMessage) => - shouldIgnoreIncomingRequest(request, config.healthPath), - }), - new ExpressInstrumentation(), - new MongoDBInstrumentation(), - new MongooseInstrumentation(), - new IORedisInstrumentation(), - new UndiciInstrumentation({ - startSpanHook: (request: UndiciRequestInfo) => getSanitizedUndiciUrlAttributes(request), - }), - ], + instrumentations, }; return new NodeSDK(sdkConfig);