📡 refactor: Gate Noisy Redis OTEL Instrumentation (#13764)

* fix(telemetry): gate noisy database instrumentation

* test(telemetry): assert opt-in instrumentation baseline

* style(telemetry): sort imports

* fix(telemetry): preserve mongoose internal tracing by default

* fix(telemetry): limit database tracing flag to redis

.env.example

@@ -138,6 +138,8 @@ NODE_MAX_OLD_SPACE_SIZE=6144
 # OTEL_TRACES_SAMPLER=parentbased_always_on
 # OTEL_LOG_LEVEL=INFO
 # OTEL_SDK_DISABLED=false
+# Enable Redis command-level spans. Disabled by default to keep backend traces high-level.
+# OTEL_IOREDIS_TRACING_ENABLED=false
 
 #===============================#
 # Real User Monitoring (Browser) #

packages/api/src/telemetry/config.spec.ts

@@ -8,6 +8,7 @@ describe('getTelemetryConfig', () => {
     expect(config.sdkDisabled).toBe(false);
     expect(config.serviceName).toBe('librechat');
     expect(config.healthPath).toBe('/health');
+    expect(config.ioredisTracingEnabled).toBe(false);
   });
 
   it('enables tracing only when OTEL_TRACING_ENABLED is true', () => {
@@ -43,4 +44,12 @@ describe('getTelemetryConfig', () => {
 
     expect(config.serviceVersion).toBe('0.8.5');
   });
+
+  it('enables ioredis instrumentation only when explicitly configured', () => {
+    const config = getTelemetryConfig({
+      OTEL_IOREDIS_TRACING_ENABLED: 'true',
+    });
+
+    expect(config.ioredisTracingEnabled).toBe(true);
+  });
 });

packages/api/src/telemetry/config.ts

@@ -6,6 +6,7 @@ export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | '
 export interface TelemetryConfig {
   enabled: boolean;
   healthPath: string;
+  ioredisTracingEnabled: boolean;
   sdkDisabled: boolean;
   serviceName: string;
   serviceVersion?: string;
@@ -32,10 +33,12 @@ export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): Teleme
   const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;
   const serviceVersion =
     normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);
+  const ioredisTracingEnabled = isTruthy(env.OTEL_IOREDIS_TRACING_ENABLED);
 
   return {
     enabled,
     serviceName,
+    ioredisTracingEnabled,
     sdkDisabled,
     serviceVersion,
     healthPath: DEFAULT_HEALTH_PATH,

packages/api/src/telemetry/sdk.spec.ts

@@ -1,8 +1,8 @@
 import { Socket } from 'node:net';
 import { IncomingMessage } from 'node:http';
 import { Agent as HttpsAgent } from 'node:https';
-import type { Span } from '@opentelemetry/api';
 import type { RequestOptions } from 'node:http';
+import type { Span } from '@opentelemetry/api';
 
 interface HttpInstrumentationOptions {
   requestHook?: (span: Span, request: object) => void;
@@ -191,6 +191,20 @@ describe('telemetry SDK lifecycle', () => {
     expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1);
     expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1);
     expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockIORedisInstrumentation).not.toHaveBeenCalled();
+    expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1);
+  });
+
+  it('enables ioredis instrumentation when explicitly configured', () => {
+    initializeTelemetry({
+      OTEL_IOREDIS_TRACING_ENABLED: 'true',
+      OTEL_TRACING_ENABLED: 'true',
+    });
+
+    expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockHttpInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1);
     expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1);
     expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1);
   });

packages/api/src/telemetry/sdk.ts

@@ -12,8 +12,8 @@ import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node';
 import type { Span, Attributes } from '@opentelemetry/api';
 import type { RequestOptions } from 'node:http';
 import type { TelemetryConfig, TelemetryStatus } from './config';
-import { getTelemetryConfig } from './config';
 import { registerShutdownTask } from '../app/shutdown';
+import { getTelemetryConfig } from './config';
 
 export interface TelemetryController {
   readonly enabled: boolean;
@@ -297,34 +297,39 @@ function getResourceAttributes(config: TelemetryConfig): Attributes {
 }
 
 function createSdk(config: TelemetryConfig): NodeSDK {
+  const instrumentations: NodeSDKConfiguration['instrumentations'] = [
+    new HttpInstrumentation({
+      headersToSpanAttributes: {
+        client: { requestHeaders: [], responseHeaders: [] },
+        server: { requestHeaders: [], responseHeaders: [] },
+      },
+      requestHook: (span: Span, request: object) => {
+        if (request instanceof IncomingMessage) {
+          requestSpans.set(request, span);
+        }
+      },
+      startIncomingSpanHook: (request: IncomingMessage) =>
+        getSanitizedIncomingUrlAttributes(request, config.healthPath),
+      startOutgoingSpanHook: (request: RequestOptions) =>
+        getSanitizedOutgoingHttpUrlAttributes(request),
+      ignoreIncomingRequestHook: (request: IncomingMessage) =>
+        shouldIgnoreIncomingRequest(request, config.healthPath),
+    }),
+    new ExpressInstrumentation(),
+    new MongoDBInstrumentation(),
+    new MongooseInstrumentation(),
+    new UndiciInstrumentation({
+      startSpanHook: (request: UndiciRequestInfo) => getSanitizedUndiciUrlAttributes(request),
+    }),
+  ];
+
+  if (config.ioredisTracingEnabled) {
+    instrumentations.push(new IORedisInstrumentation());
+  }
+
   const sdkConfig: Partial<NodeSDKConfiguration> = {
     resource: resourceFromAttributes(getResourceAttributes(config)),
-    instrumentations: [
-      new HttpInstrumentation({
-        headersToSpanAttributes: {
-          client: { requestHeaders: [], responseHeaders: [] },
-          server: { requestHeaders: [], responseHeaders: [] },
-        },
-        requestHook: (span: Span, request: object) => {
-          if (request instanceof IncomingMessage) {
-            requestSpans.set(request, span);
-          }
-        },
-        startIncomingSpanHook: (request: IncomingMessage) =>
-          getSanitizedIncomingUrlAttributes(request, config.healthPath),
-        startOutgoingSpanHook: (request: RequestOptions) =>
-          getSanitizedOutgoingHttpUrlAttributes(request),
-        ignoreIncomingRequestHook: (request: IncomingMessage) =>
-          shouldIgnoreIncomingRequest(request, config.healthPath),
-      }),
-      new ExpressInstrumentation(),
-      new MongoDBInstrumentation(),
-      new MongooseInstrumentation(),
-      new IORedisInstrumentation(),
-      new UndiciInstrumentation({
-        startSpanHook: (request: UndiciRequestInfo) => getSanitizedUndiciUrlAttributes(request),
-      }),
-    ],
+    instrumentations,
   };
 
   return new NodeSDK(sdkConfig);