📡 feat: Add Backend OpenTelemetry Tracing (#12909)

* feat: add backend OpenTelemetry tracing * fix: address telemetry type checks * fix: mark aborted telemetry requests as errors * fix: record telemetry identity after auth * fix: avoid forced telemetry signal exit * fix: harden telemetry request attribution * fix: record telemetry errors on request span * chore: order imports and reorganize middleware usage * fix: reduce telemetry startup overhead * fix: preserve live telemetry controller state * fix: redact telemetry URL attributes
2026-08-27 04:07:05 +00:00 · 2026-05-14 09:08:55 -04:00 · 2026-05-14 09:08:55 -04:00 · 050b7fd43a
commit 050b7fd43a
parent 7e4c5d9ded
16 changed files with 1864 additions and 8 deletions
--- a/packages/api/src/telemetry/config.spec.ts
+++ b/packages/api/src/telemetry/config.spec.ts
@ -0,0 +1,46 @@
+import { getTelemetryConfig } from './config';
+
+describe('getTelemetryConfig', () => {
+  it('defaults tracing off', () => {
+    const config = getTelemetryConfig({});
+
+    expect(config.enabled).toBe(false);
+    expect(config.sdkDisabled).toBe(false);
+    expect(config.serviceName).toBe('librechat');
+    expect(config.healthPath).toBe('/health');
+  });
+
+  it('enables tracing only when OTEL_TRACING_ENABLED is true', () => {
+    expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'true' }).enabled).toBe(true);
+    expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'TRUE' }).enabled).toBe(true);
+    expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'false' }).enabled).toBe(false);
+  });
+
+  it('lets OTEL_SDK_DISABLED override tracing enablement', () => {
+    const config = getTelemetryConfig({
+      OTEL_SDK_DISABLED: 'true',
+      OTEL_TRACING_ENABLED: 'true',
+    });
+
+    expect(config.enabled).toBe(false);
+    expect(config.sdkDisabled).toBe(true);
+  });
+
+  it('uses standard service env vars when provided', () => {
+    const config = getTelemetryConfig({
+      OTEL_SERVICE_NAME: ' librechat-api ',
+      OTEL_SERVICE_VERSION: ' 1.2.3 ',
+    });
+
+    expect(config.serviceName).toBe('librechat-api');
+    expect(config.serviceVersion).toBe('1.2.3');
+  });
+
+  it('falls back to npm package version when service version is absent', () => {
+    const config = getTelemetryConfig({
+      npm_package_version: '0.8.5',
+    });
+
+    expect(config.serviceVersion).toBe('0.8.5');
+  });
+});
--- a/packages/api/src/telemetry/config.ts
+++ b/packages/api/src/telemetry/config.ts
@ -0,0 +1,43 @@
+const DEFAULT_SERVICE_NAME = 'librechat';
+export const DEFAULT_HEALTH_PATH = '/health';
+
+export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | 'stopped';
+
+export interface TelemetryConfig {
+  enabled: boolean;
+  healthPath: string;
+  sdkDisabled: boolean;
+  serviceName: string;
+  serviceVersion?: string;
+}
+
+function isTruthy(value?: string | boolean | null): boolean {
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  if (typeof value === 'string') {
+    return value.trim().toLowerCase() === 'true';
+  }
+  return false;
+}
+
+function normalizeEnvValue(value?: string): string | undefined {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : undefined;
+}
+
+export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): TelemetryConfig {
+  const sdkDisabled = isTruthy(env.OTEL_SDK_DISABLED);
+  const enabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled;
+  const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;
+  const serviceVersion =
+    normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);
+
+  return {
+    enabled,
+    serviceName,
+    sdkDisabled,
+    serviceVersion,
+    healthPath: DEFAULT_HEALTH_PATH,
+  };
+}
--- a/packages/api/src/telemetry/index.ts
+++ b/packages/api/src/telemetry/index.ts
@ -0,0 +1,5 @@
+export { getTelemetryConfig } from './config';
+export { initializeTelemetry, shutdownTelemetry } from './sdk';
+export { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
+export type { TelemetryConfig, TelemetryStatus } from './config';
+export type { TelemetryController } from './sdk';
--- a/packages/api/src/telemetry/middleware.spec.ts
+++ b/packages/api/src/telemetry/middleware.spec.ts
@ -0,0 +1,395 @@
+import { EventEmitter } from 'node:events';
+import { SpanStatusCode, trace } from '@opentelemetry/api';
+import type { NextFunction, Response } from 'express';
+import type { Span } from '@opentelemetry/api';
+import type { ServerRequest } from '~/types';
+import { getTelemetryRequestSpan } from './sdk';
+import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
+
+jest.mock('./sdk', () => ({
+  getTelemetryRequestSpan: jest.fn(),
+}));
+
+const mockGetTelemetryRequestSpan = getTelemetryRequestSpan as jest.MockedFunction<
+  typeof getTelemetryRequestSpan
+>;
+
+interface MockResponse extends EventEmitter {
+  statusCode: number;
+  writableEnded: boolean;
+}
+
+function createSpan(): jest.Mocked<Span> {
+  const span = {} as jest.Mocked<Span>;
+  span.addEvent = jest.fn<jest.Mocked<Span>, Parameters<Span['addEvent']>>(() => span);
+  span.addLink = jest.fn<jest.Mocked<Span>, Parameters<Span['addLink']>>(() => span);
+  span.addLinks = jest.fn<jest.Mocked<Span>, Parameters<Span['addLinks']>>(() => span);
+  span.end = jest.fn<void, Parameters<Span['end']>>();
+  span.isRecording = jest.fn<boolean, Parameters<Span['isRecording']>>(() => true);
+  span.recordException = jest.fn<void, Parameters<Span['recordException']>>();
+  span.setAttribute = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttribute']>>(() => span);
+  span.setAttributes = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttributes']>>(() => span);
+  span.setStatus = jest.fn<jest.Mocked<Span>, Parameters<Span['setStatus']>>(() => span);
+  span.spanContext = jest.fn<ReturnType<Span['spanContext']>, Parameters<Span['spanContext']>>(
+    () => ({
+      spanId: '0000000000000000',
+      traceFlags: 0,
+      traceId: '00000000000000000000000000000000',
+    }),
+  );
+  span.updateName = jest.fn<jest.Mocked<Span>, Parameters<Span['updateName']>>(() => span);
+  return span;
+}
+
+function createResponse(statusCode = 200): MockResponse {
+  const res = new EventEmitter() as MockResponse;
+  res.statusCode = statusCode;
+  res.writableEnded = false;
+  return res;
+}
+
+function createRequest(overrides: Partial<ServerRequest> = {}): ServerRequest {
+  return {
+    baseUrl: '/api/messages',
+    body: {
+      prompt: 'do not capture this prompt',
+      text: 'do not capture this body',
+    },
+    headers: {
+      authorization: 'Bearer do-not-capture-this-auth-header',
+      cookie: 'session=do-not-capture-this-cookie',
+      'x-api-key': 'do-not-capture-this-api-key',
+    },
+    method: 'POST',
+    path: '/api/messages/conversation-1',
+    route: { path: '/:conversationId' },
+    user: {
+      email: 'do-not-capture@example.com',
+      id: 'user-1',
+      tenantId: 'tenant-1',
+    } as ServerRequest['user'],
+    ...overrides,
+  } as ServerRequest;
+}
+
+afterEach(() => {
+  mockGetTelemetryRequestSpan.mockReset();
+});
+
+describe('telemetryMiddleware', () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it('passes through without an active span', () => {
+    const next = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);
+
+    telemetryMiddleware(createRequest(), createResponse() as Response, next);
+
+    expect(next).toHaveBeenCalledTimes(1);
+  });
+
+  it('uses the stored request span for deferred completion attributes', () => {
+    const activeSpan = createSpan();
+    const requestSpan = createSpan();
+    const req = createRequest();
+    const res = createResponse(202);
+    const next: NextFunction = jest.fn();
+    mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);
+
+    telemetryMiddleware(req, res as Response, next);
+    res.emit('finish');
+
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(trace.getActiveSpan).not.toHaveBeenCalled();
+    expect(activeSpan.setAttributes).not.toHaveBeenCalled();
+    expect(requestSpan.setAttributes).toHaveBeenCalledWith({
+      'http.request.method': 'POST',
+    });
+    expect(requestSpan.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'http.response.status_code': 202,
+        'http.route': '/api/messages/:conversationId',
+      }),
+    );
+  });
+
+  it('records safe route and identity attributes without body content', () => {
+    const span = createSpan();
+    const req = createRequest();
+    const res = createResponse(201);
+    const next: NextFunction = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(req, res as Response, next);
+    res.emit('finish');
+
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(span.setAttributes).toHaveBeenCalledWith({
+      'http.request.method': 'POST',
+    });
+    expect(span.setAttributes).toHaveBeenCalledWith({
+      'enduser.id': 'user-1',
+      'librechat.tenant.id': 'tenant-1',
+    });
+    expect(span.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'http.response.status_code': 201,
+        'http.route': '/api/messages/:conversationId',
+      }),
+    );
+
+    const capturedAttributes = JSON.stringify(span.setAttributes.mock.calls);
+    expect(capturedAttributes).not.toContain('do not capture this prompt');
+    expect(capturedAttributes).not.toContain('do not capture this body');
+    expect(capturedAttributes).not.toContain('do-not-capture@example.com');
+    expect(capturedAttributes).not.toContain('conversation-1');
+    expect(capturedAttributes).not.toContain('do-not-capture-this-auth-header');
+    expect(capturedAttributes).not.toContain('do-not-capture-this-cookie');
+    expect(capturedAttributes).not.toContain('do-not-capture-this-api-key');
+  });
+
+  it('records identity attributes populated by downstream middleware', () => {
+    const span = createSpan();
+    const req = createRequest({
+      headers: {},
+      user: undefined,
+    });
+    const res = createResponse(200);
+    const next: NextFunction = jest.fn(() => {
+      req.user = {
+        id: 'late-user',
+        tenantId: 'late-tenant',
+      } as ServerRequest['user'];
+    });
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(req, res as Response, next);
+
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(span.setAttributes).toHaveBeenCalledTimes(1);
+
+    res.emit('finish');
+
+    expect(span.setAttributes).toHaveBeenCalledWith({
+      'enduser.id': 'late-user',
+      'librechat.tenant.id': 'late-tenant',
+    });
+  });
+
+  it('does not derive tenant identity from request headers', () => {
+    const span = createSpan();
+    const req = createRequest({
+      headers: {
+        'x-tenant-id': 'spoofed-tenant',
+      },
+      user: undefined,
+    });
+    const res = createResponse(200);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(req, res as Response, jest.fn());
+    res.emit('finish');
+
+    expect(span.setAttributes).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        'librechat.tenant.id': 'spoofed-tenant',
+      }),
+    );
+    expect(JSON.stringify(span.setAttributes.mock.calls)).not.toContain('spoofed-tenant');
+  });
+
+  it('ignores health checks', () => {
+    const span = createSpan();
+    const next = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(
+      createRequest({
+        baseUrl: '',
+        path: '/health',
+        route: undefined,
+      }),
+      createResponse() as Response,
+      next,
+    );
+
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(span.setAttributes).not.toHaveBeenCalled();
+  });
+
+  it('uses a low-cardinality fallback for unmatched API routes', () => {
+    const span = createSpan();
+    const req = createRequest({
+      baseUrl: '',
+      path: '/api/nonexistent/123',
+      route: undefined,
+    });
+    const res = createResponse(404);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(req, res as Response, jest.fn());
+    res.emit('finish');
+
+    expect(span.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'http.route': '/api/*',
+        'http.response.status_code': 404,
+      }),
+    );
+  });
+
+  it('uses a low-cardinality fallback for unmatched SPA routes', () => {
+    const span = createSpan();
+    const req = createRequest({
+      baseUrl: '',
+      path: '/chat/conversation-id',
+      route: undefined,
+    });
+    const res = createResponse(200);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(req, res as Response, jest.fn());
+    res.emit('finish');
+
+    expect(span.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'http.route': 'spa_fallback',
+        'http.response.status_code': 200,
+      }),
+    );
+  });
+
+  it('marks server responses as errored', () => {
+    const span = createSpan();
+    const req = createRequest();
+    const res = createResponse(500);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(req, res as Response, jest.fn());
+    res.emit('finish');
+
+    expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+  });
+
+  it('records completion attributes only once when finish and close both fire', () => {
+    const span = createSpan();
+    const res = createResponse(200);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(createRequest(), res as Response, jest.fn());
+    res.emit('finish');
+    res.emit('close');
+
+    expect(span.setAttributes).toHaveBeenCalledTimes(3);
+  });
+
+  it('marks client disconnects before finish as aborted errors', () => {
+    const span = createSpan();
+    const res = createResponse(200);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryMiddleware(createRequest(), res as Response, jest.fn());
+    res.emit('close');
+
+    expect(span.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'http.response.status_code': 499,
+        'librechat.request.aborted': true,
+      }),
+    );
+    expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+  });
+});
+
+describe('telemetryErrorMiddleware', () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it('records exceptions and forwards the error', () => {
+    const span = createSpan();
+    const error = new TypeError('boom');
+    const next = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
+
+    expect(span.recordException).toHaveBeenCalledWith(error);
+    expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+    expect(span.setAttributes).toHaveBeenCalledWith({
+      'enduser.id': 'user-1',
+      'librechat.tenant.id': 'tenant-1',
+    });
+    expect(span.setAttributes).toHaveBeenCalledWith({
+      'error.type': 'TypeError',
+      'http.route': '/api/messages/:conversationId',
+    });
+    expect(next).toHaveBeenCalledWith(error);
+  });
+
+  it('records exceptions on the stored request span when available', () => {
+    const activeSpan = createSpan();
+    const requestSpan = createSpan();
+    const error = new TypeError('boom');
+    const next = jest.fn();
+    mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);
+
+    telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
+
+    expect(trace.getActiveSpan).not.toHaveBeenCalled();
+    expect(activeSpan.recordException).not.toHaveBeenCalled();
+    expect(requestSpan.recordException).toHaveBeenCalledWith(error);
+    expect(requestSpan.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+    expect(next).toHaveBeenCalledWith(error);
+  });
+
+  it('handles non-Error values without throwing', () => {
+    const span = createSpan();
+    const next = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryErrorMiddleware('boom', createRequest(), createResponse() as Response, next);
+
+    expect(span.recordException).toHaveBeenCalledWith('boom');
+    expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+    expect(span.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'error.type': 'string',
+        'http.route': '/api/messages/:conversationId',
+      }),
+    );
+    expect(next).toHaveBeenCalledWith('boom');
+  });
+
+  it('handles null error values without throwing', () => {
+    const span = createSpan();
+    const next = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+    telemetryErrorMiddleware(null, createRequest(), createResponse() as Response, next);
+
+    expect(span.recordException).not.toHaveBeenCalled();
+    expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+    expect(span.setAttributes).toHaveBeenCalledWith(
+      expect.objectContaining({
+        'error.type': 'null',
+        'http.route': '/api/messages/:conversationId',
+      }),
+    );
+    expect(next).toHaveBeenCalledWith(null);
+  });
+
+  it('forwards the error without an active span', () => {
+    const error = new Error('boom');
+    const next = jest.fn();
+    jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);
+
+    telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
+
+    expect(next).toHaveBeenCalledWith(error);
+  });
+});
--- a/packages/api/src/telemetry/middleware.ts
+++ b/packages/api/src/telemetry/middleware.ts
@ -0,0 +1,171 @@
+import { SpanStatusCode, trace } from '@opentelemetry/api';
+import type { Span, Attributes } from '@opentelemetry/api';
+import type { NextFunction, Response } from 'express';
+import type { ServerRequest } from '~/types';
+import { getTelemetryRequestSpan } from './sdk';
+import { DEFAULT_HEALTH_PATH } from './config';
+
+const CLIENT_CLOSED_REQUEST_STATUS_CODE = 499;
+
+type ExpressErrorValue =
+  | Error
+  | string
+  | number
+  | boolean
+  | bigint
+  | symbol
+  | object
+  | null
+  | undefined;
+
+function getUserId(req: ServerRequest): string | undefined {
+  return req.user?.id;
+}
+
+function getTenantId(req: ServerRequest): string | undefined {
+  return req.user?.tenantId;
+}
+
+function isHealthPath(req: ServerRequest): boolean {
+  return req.path === DEFAULT_HEALTH_PATH;
+}
+
+function isApiPath(req: ServerRequest): boolean {
+  return req.path === '/api' || req.path.startsWith('/api/');
+}
+
+function getRoutePath(req: ServerRequest): string {
+  const routePath = req.route?.path;
+  if (typeof routePath === 'string') {
+    return `${req.baseUrl}${routePath}`;
+  }
+
+  if (isHealthPath(req)) {
+    return '/health';
+  }
+
+  if (isApiPath(req)) {
+    return '/api/*';
+  }
+
+  return 'spa_fallback';
+}
+
+function setIdentityAttributes(span: Span, req: ServerRequest): void {
+  const userId = getUserId(req);
+  const tenantId = getTenantId(req);
+
+  if (!userId && !tenantId) {
+    return;
+  }
+
+  const attributes: Attributes = {};
+
+  if (userId) {
+    attributes['enduser.id'] = userId;
+  }
+
+  if (tenantId) {
+    attributes['librechat.tenant.id'] = tenantId;
+  }
+
+  span.setAttributes(attributes);
+}
+
+function setCompletionAttributes(
+  span: Span,
+  req: ServerRequest,
+  res: Response,
+  aborted = false,
+): void {
+  const statusCode = aborted ? CLIENT_CLOSED_REQUEST_STATUS_CODE : res.statusCode;
+  const routePath = getRoutePath(req);
+  const attributes: Attributes = {
+    'http.route': routePath,
+    'http.response.status_code': statusCode,
+  };
+
+  if (aborted) {
+    attributes['librechat.request.aborted'] = true;
+  }
+
+  setIdentityAttributes(span, req);
+  span.setAttributes(attributes);
+
+  if (aborted || statusCode >= 500) {
+    span.setStatus({ code: SpanStatusCode.ERROR });
+  }
+}
+
+export function telemetryMiddleware(req: ServerRequest, res: Response, next: NextFunction): void {
+  if (isHealthPath(req)) {
+    next();
+    return;
+  }
+
+  const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();
+  if (!span) {
+    next();
+    return;
+  }
+
+  span.setAttributes({
+    'http.request.method': req.method,
+  });
+
+  let completed = false;
+  const complete = () => {
+    if (completed) {
+      return;
+    }
+    completed = true;
+    setCompletionAttributes(span, req, res);
+  };
+
+  const close = () => {
+    if (completed) {
+      return;
+    }
+    completed = true;
+    setCompletionAttributes(span, req, res, !res.writableEnded);
+  };
+
+  res.once('finish', complete);
+  res.once('close', close);
+  next();
+}
+
+export function telemetryErrorMiddleware(
+  err: ExpressErrorValue,
+  req: ServerRequest,
+  _res: Response,
+  next: NextFunction,
+): void {
+  const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();
+  if (span) {
+    const routePath = getRoutePath(req);
+    if (err) {
+      span.recordException(err instanceof Error ? err : String(err));
+    }
+    span.setStatus({ code: SpanStatusCode.ERROR });
+    setIdentityAttributes(span, req);
+    span.setAttributes({
+      'error.type': getErrorType(err),
+      'http.route': routePath,
+    });
+  }
+
+  next(err);
+}
+
+function getErrorType(err: ExpressErrorValue): string {
+  if (err instanceof Error) {
+    return err.name || err.constructor.name;
+  }
+
+  if (err === null) {
+    return 'null';
+  }
+
+  return typeof err;
+}
--- a/packages/api/src/telemetry/sdk.spec.ts
+++ b/packages/api/src/telemetry/sdk.spec.ts
@ -0,0 +1,357 @@
+import { Socket } from 'node:net';
+import { IncomingMessage } from 'node:http';
+import type { Span } from '@opentelemetry/api';
+
+interface HttpInstrumentationOptions {
+  requestHook?: (span: Span, request: object) => void;
+  startIncomingSpanHook?: (request: IncomingMessage) => Record<string, string>;
+}
+
+const mockStart = jest.fn();
+const mockShutdown = jest.fn();
+const mockNodeSDK = jest.fn(() => ({
+  start: mockStart,
+  shutdown: mockShutdown,
+}));
+const mockExpressInstrumentation = jest.fn(() => ({ name: 'express' }));
+const mockHttpInstrumentation = jest.fn((options?: HttpInstrumentationOptions) => ({
+  name: 'http',
+  options,
+}));
+const mockIORedisInstrumentation = jest.fn(() => ({ name: 'ioredis' }));
+const mockMongoDBInstrumentation = jest.fn(() => ({ name: 'mongodb' }));
+const mockMongooseInstrumentation = jest.fn(() => ({ name: 'mongoose' }));
+const mockUndiciInstrumentation = jest.fn(() => ({ name: 'undici' }));
+const mockResourceFromAttributes = jest.fn((attributes: object) => ({ attributes }));
+
+jest.mock(
+  '@opentelemetry/sdk-node',
+  () => ({
+    NodeSDK: mockNodeSDK,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/instrumentation-express',
+  () => ({
+    ExpressInstrumentation: mockExpressInstrumentation,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/instrumentation-http',
+  () => ({
+    HttpInstrumentation: mockHttpInstrumentation,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/instrumentation-ioredis',
+  () => ({
+    IORedisInstrumentation: mockIORedisInstrumentation,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/instrumentation-mongodb',
+  () => ({
+    MongoDBInstrumentation: mockMongoDBInstrumentation,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/instrumentation-mongoose',
+  () => ({
+    MongooseInstrumentation: mockMongooseInstrumentation,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/instrumentation-undici',
+  () => ({
+    UndiciInstrumentation: mockUndiciInstrumentation,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/resources',
+  () => ({
+    resourceFromAttributes: mockResourceFromAttributes,
+  }),
+  { virtual: true },
+);
+
+jest.mock(
+  '@opentelemetry/semantic-conventions',
+  () => ({
+    ATTR_SERVICE_NAME: 'service.name',
+    ATTR_SERVICE_VERSION: 'service.version',
+  }),
+  { virtual: true },
+);
+
+async function flushSignalShutdown(): Promise<void> {
+  await new Promise((resolve) => setImmediate(resolve));
+}
+
+describe('telemetry SDK lifecycle', () => {
+  let emitWarningSpy: jest.SpyInstance;
+  let getTelemetryRequestSpan: (typeof import('./sdk'))['getTelemetryRequestSpan'];
+  let initializeTelemetry: (typeof import('./sdk'))['initializeTelemetry'];
+  let resetTelemetryForTests: (typeof import('./sdk'))['resetTelemetryForTests'];
+  let shutdownTelemetry: (typeof import('./sdk'))['shutdownTelemetry'];
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+    ({ getTelemetryRequestSpan, initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } =
+      await import('./sdk'));
+    await resetTelemetryForTests();
+    Reflect.deleteProperty(globalThis, 'Bun');
+    emitWarningSpy = jest.spyOn(process, 'emitWarning').mockImplementation(() => true);
+  });
+
+  afterEach(async () => {
+    await resetTelemetryForTests();
+    emitWarningSpy.mockRestore();
+    Reflect.deleteProperty(globalThis, 'Bun');
+  });
+
+  it('does not initialize when tracing is disabled by default', () => {
+    const controller = initializeTelemetry({});
+
+    expect(controller.enabled).toBe(false);
+    expect(controller.status).toBe('disabled');
+    expect(mockNodeSDK).not.toHaveBeenCalled();
+  });
+
+  it('does not initialize when OTEL_SDK_DISABLED is true', () => {
+    const controller = initializeTelemetry({
+      OTEL_SDK_DISABLED: 'true',
+      OTEL_TRACING_ENABLED: 'true',
+    });
+
+    expect(controller.enabled).toBe(false);
+    expect(controller.status).toBe('disabled');
+    expect(mockNodeSDK).not.toHaveBeenCalled();
+  });
+
+  it('does not initialize under Bun runtime', () => {
+    Object.defineProperty(globalThis, 'Bun', {
+      configurable: true,
+      value: {},
+    });
+
+    const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    expect(controller.enabled).toBe(false);
+    expect(controller.status).toBe('disabled');
+    expect(mockNodeSDK).not.toHaveBeenCalled();
+  });
+
+  it('starts the Node SDK once when enabled', () => {
+    const first = initializeTelemetry({
+      OTEL_SERVICE_NAME: 'librechat-test',
+      OTEL_TRACING_ENABLED: 'true',
+    });
+    const second = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    expect(first.enabled).toBe(true);
+    expect(first.status).toBe('started');
+    expect(second.enabled).toBe(true);
+    expect(mockNodeSDK).toHaveBeenCalledTimes(1);
+    expect(mockStart).toHaveBeenCalledTimes(1);
+    expect(mockResourceFromAttributes).toHaveBeenCalledWith({
+      'service.name': 'librechat-test',
+    });
+    expect(mockHttpInstrumentation).toHaveBeenCalledWith(
+      expect.objectContaining({
+        headersToSpanAttributes: {
+          client: { requestHeaders: [], responseHeaders: [] },
+          server: { requestHeaders: [], responseHeaders: [] },
+        },
+      }),
+    );
+    expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1);
+    expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1);
+  });
+
+  it('tracks HTTP server request spans for completion updates', () => {
+    const span = {} as Span;
+    const request = new IncomingMessage(new Socket());
+    initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+    const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
+    const requestHook = instrumentationOptions?.requestHook;
+
+    if (!requestHook) {
+      throw new Error('HTTP instrumentation requestHook was not configured');
+    }
+
+    requestHook(span, request);
+
+    expect(getTelemetryRequestSpan(request)).toBe(span);
+  });
+
+  it('redacts incoming URL attributes before HTTP spans are exported', () => {
+    const request = new IncomingMessage(new Socket());
+    request.url = '/oauth/callback?code=secret-code&state=secret-state';
+    initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+    const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
+    const startIncomingSpanHook = instrumentationOptions?.startIncomingSpanHook;
+
+    if (!startIncomingSpanHook) {
+      throw new Error('HTTP instrumentation startIncomingSpanHook was not configured');
+    }
+
+    const attributes = startIncomingSpanHook(request);
+
+    expect(attributes).toEqual({
+      'http.target': 'spa_fallback?[REDACTED]',
+      'http.url': 'spa_fallback?[REDACTED]',
+      'url.full': 'spa_fallback?[REDACTED]',
+      'url.path': 'spa_fallback',
+      'url.query': '[REDACTED]',
+    });
+    expect(JSON.stringify(attributes)).not.toContain('secret-code');
+    expect(JSON.stringify(attributes)).not.toContain('secret-state');
+  });
+
+  it('reflects lifecycle status from the controller getter', async () => {
+    const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    expect(controller.status).toBe('started');
+    await controller.shutdown();
+    expect(controller.status).toBe('stopped');
+    expect(controller.enabled).toBe(false);
+  });
+
+  it('handles async SDK start failures without throwing', async () => {
+    mockStart.mockRejectedValueOnce(new Error('async start failed'));
+
+    const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    expect(controller.enabled).toBe(true);
+    expect(controller.status).toBe('starting');
+    await controller.shutdown();
+    expect(controller.enabled).toBe(false);
+    expect(controller.status).toBe('failed');
+    expect(emitWarningSpy).toHaveBeenCalledWith(
+      'OpenTelemetry initialization failed: async start failed',
+      { code: 'LIBRECHAT_OTEL' },
+    );
+  });
+
+  it('returns failed status without throwing when SDK start fails', () => {
+    mockStart.mockImplementationOnce(() => {
+      throw new Error('start failed');
+    });
+
+    const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    expect(controller.enabled).toBe(false);
+    expect(controller.status).toBe('failed');
+    expect(emitWarningSpy).toHaveBeenCalledWith(
+      'OpenTelemetry initialization failed: start failed',
+      { code: 'LIBRECHAT_OTEL' },
+    );
+  });
+
+  it('shuts down the active SDK idempotently', async () => {
+    initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+    await shutdownTelemetry();
+    await shutdownTelemetry();
+
+    expect(mockShutdown).toHaveBeenCalledTimes(1);
+  });
+
+  it('coalesces concurrent shutdown calls', async () => {
+    let resolveShutdown: () => void = () => undefined;
+    mockShutdown.mockReturnValueOnce(
+      new Promise<void>((resolve) => {
+        resolveShutdown = resolve;
+      }),
+    );
+    initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    const firstShutdown = shutdownTelemetry();
+    const secondShutdown = shutdownTelemetry();
+    expect(mockShutdown).toHaveBeenCalledTimes(1);
+
+    resolveShutdown();
+    await Promise.all([firstShutdown, secondShutdown]);
+  });
+
+  it('keeps the active SDK available when shutdown fails', async () => {
+    mockShutdown.mockRejectedValueOnce(new Error('shutdown failed'));
+    const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    await expect(shutdownTelemetry()).rejects.toThrow('shutdown failed');
+    expect(controller.status).toBe('started');
+
+    await shutdownTelemetry();
+    expect(mockShutdown).toHaveBeenCalledTimes(2);
+    expect(controller.status).toBe('stopped');
+  });
+
+  it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(
+    'does not force process exit when another %s handler is registered',
+    async (signal) => {
+      const otherHandler = jest.fn();
+      const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+      initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+      process.once(signal, otherHandler);
+
+      process.emit(signal, signal);
+      await flushSignalShutdown();
+
+      expect(mockShutdown).toHaveBeenCalledTimes(1);
+      expect(otherHandler).toHaveBeenCalledTimes(1);
+      expect(killSpy).not.toHaveBeenCalled();
+
+      killSpy.mockRestore();
+    },
+  );
+
+  it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(
+    'reraises the shutdown %s signal when telemetry is the only signal handler',
+    async (signal) => {
+      const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+      initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+      process.emit(signal, signal);
+      await flushSignalShutdown();
+
+      expect(mockShutdown).toHaveBeenCalledTimes(1);
+      expect(killSpy).toHaveBeenCalledWith(process.pid, signal);
+
+      killSpy.mockRestore();
+    },
+  );
+
+  it('warns and reraises the signal when shutdown rejects', async () => {
+    mockShutdown.mockRejectedValueOnce(new Error('signal shutdown failed'));
+    const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+    initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+    process.emit('SIGTERM', 'SIGTERM');
+    await flushSignalShutdown();
+
+    expect(mockShutdown).toHaveBeenCalledTimes(1);
+    expect(emitWarningSpy).toHaveBeenCalledWith(
+      'OpenTelemetry shutdown failed: signal shutdown failed',
+      { code: 'LIBRECHAT_OTEL' },
+    );
+    expect(killSpy).toHaveBeenCalledWith(process.pid, 'SIGTERM');
+
+    killSpy.mockRestore();
+  });
+});
--- a/packages/api/src/telemetry/sdk.ts
+++ b/packages/api/src/telemetry/sdk.ts
@ -0,0 +1,328 @@
+import { IncomingMessage } from 'node:http';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';
+import { MongoDBInstrumentation } from '@opentelemetry/instrumentation-mongodb';
+import { MongooseInstrumentation } from '@opentelemetry/instrumentation-mongoose';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node';
+import type { Span, Attributes } from '@opentelemetry/api';
+import type { TelemetryConfig, TelemetryStatus } from './config';
+import { getTelemetryConfig } from './config';
+
+export interface TelemetryController {
+  readonly enabled: boolean;
+  readonly status: TelemetryStatus;
+  shutdown: () => Promise<void>;
+}
+
+const WARNING_CODE = 'LIBRECHAT_OTEL';
+const REDACTED_QUERY_VALUE = '[REDACTED]';
+const SIGNAL_SHUTDOWN_TIMEOUT_MS = 5_000;
+
+interface RegisteredSignal {
+  signal: NodeJS.Signals;
+  listener: NodeJS.SignalsListener;
+}
+
+let activeSdk: NodeSDK | undefined;
+let pendingSdk: NodeSDK | undefined;
+let startPromise: Promise<void> | undefined;
+let shutdownPromise: Promise<void> | undefined;
+let status: TelemetryStatus = 'stopped';
+let registeredSignals: RegisteredSignal[] = [];
+let requestSpans = new WeakMap<IncomingMessage, Span>();
+
+function isBunRuntime(): boolean {
+  return Reflect.get(globalThis, 'Bun') != null;
+}
+
+function shouldIgnoreIncomingRequest(request: IncomingMessage, healthPath: string): boolean {
+  return request.url === healthPath || request.url?.startsWith(`${healthPath}?`) === true;
+}
+
+function getIncomingUrlInfo(request: IncomingMessage): { hasQuery: boolean; pathname: string } {
+  const rawUrl = request.url ?? '/';
+
+  try {
+    const parsedUrl = new URL(rawUrl, 'http://localhost');
+    return {
+      hasQuery: parsedUrl.search.length > 1,
+      pathname: parsedUrl.pathname || '/',
+    };
+  } catch {
+    const queryIndex = rawUrl.indexOf('?');
+    return {
+      hasQuery: queryIndex >= 0 && queryIndex < rawUrl.length - 1,
+      pathname: queryIndex >= 0 ? rawUrl.slice(0, queryIndex) || '/' : rawUrl || '/',
+    };
+  }
+}
+
+function getLowCardinalityUrlPath(pathname: string, healthPath: string): string {
+  if (pathname === healthPath) {
+    return healthPath;
+  }
+
+  if (pathname === '/api' || pathname.startsWith('/api/')) {
+    return '/api/*';
+  }
+
+  return 'spa_fallback';
+}
+
+function getSanitizedIncomingUrlAttributes(
+  request: IncomingMessage,
+  healthPath: string,
+): Attributes {
+  const { hasQuery, pathname } = getIncomingUrlInfo(request);
+  const safePath = getLowCardinalityUrlPath(pathname, healthPath);
+  const safeTarget = hasQuery ? `${safePath}?${REDACTED_QUERY_VALUE}` : safePath;
+  const attributes: Attributes = {
+    'http.target': safeTarget,
+    'http.url': safeTarget,
+    'url.full': safeTarget,
+    'url.path': safePath,
+  };
+
+  if (hasQuery) {
+    attributes['url.query'] = REDACTED_QUERY_VALUE;
+  }
+
+  return attributes;
+}
+
+function getResourceAttributes(config: TelemetryConfig): Attributes {
+  const attributes: Attributes = {
+    [ATTR_SERVICE_NAME]: config.serviceName,
+  };
+
+  if (config.serviceVersion) {
+    attributes[ATTR_SERVICE_VERSION] = config.serviceVersion;
+  }
+
+  return attributes;
+}
+
+function createSdk(config: TelemetryConfig): NodeSDK {
+  const sdkConfig: Partial<NodeSDKConfiguration> = {
+    resource: resourceFromAttributes(getResourceAttributes(config)),
+    instrumentations: [
+      new HttpInstrumentation({
+        headersToSpanAttributes: {
+          client: { requestHeaders: [], responseHeaders: [] },
+          server: { requestHeaders: [], responseHeaders: [] },
+        },
+        requestHook: (span: Span, request: object) => {
+          if (request instanceof IncomingMessage) {
+            requestSpans.set(request, span);
+          }
+        },
+        startIncomingSpanHook: (request: IncomingMessage) =>
+          getSanitizedIncomingUrlAttributes(request, config.healthPath),
+        ignoreIncomingRequestHook: (request: IncomingMessage) =>
+          shouldIgnoreIncomingRequest(request, config.healthPath),
+      }),
+      new ExpressInstrumentation(),
+      new MongoDBInstrumentation(),
+      new MongooseInstrumentation(),
+      new IORedisInstrumentation(),
+      new UndiciInstrumentation(),
+    ],
+  };
+
+  return new NodeSDK(sdkConfig);
+}
+
+export function getTelemetryRequestSpan(request: IncomingMessage): Span | undefined {
+  return requestSpans.get(request);
+}
+
+/**
+ * NodeSDK.start has been synchronous in some supported OpenTelemetry versions
+ * and promise-returning in others, so the lifecycle wrapper accepts either form.
+ */
+function startSdk(sdk: NodeSDK): void | Promise<void> {
+  return (sdk as NodeSDK & { start: () => void | Promise<void> }).start();
+}
+
+function emitWarning(message: string): void {
+  process.emitWarning(message, { code: WARNING_CODE });
+}
+
+function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function isControllerEnabled(): boolean {
+  return status === 'starting' || status === 'started';
+}
+
+function makeController(): TelemetryController {
+  return {
+    get enabled() {
+      return isControllerEnabled();
+    },
+    get status() {
+      return status;
+    },
+    shutdown: shutdownTelemetry,
+  };
+}
+
+function unregisterShutdownHandlers(): void {
+  for (const { signal, listener } of registeredSignals) {
+    process.removeListener(signal, listener);
+  }
+  registeredSignals = [];
+}
+
+function registerShutdownHandlers(): void {
+  if (registeredSignals.length > 0) {
+    return;
+  }
+
+  const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT'];
+  registeredSignals = signals.map((signal) => {
+    const listener: NodeJS.SignalsListener = () => {
+      const shouldReraiseSignal = process.listenerCount(signal) === 0;
+      withTimeout(shutdownTelemetry(), SIGNAL_SHUTDOWN_TIMEOUT_MS)
+        .catch((error) => {
+          emitWarning(`OpenTelemetry shutdown failed: ${getErrorMessage(error)}`);
+        })
+        .finally(() => {
+          if (shouldReraiseSignal) {
+            process.kill(process.pid, signal);
+          }
+        });
+    };
+    process.once(signal, listener);
+    return { signal, listener };
+  });
+}
+
+function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  let timeout: NodeJS.Timeout | undefined;
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => {
+      reject(new Error(`timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    timeout.unref?.();
+  });
+
+  return Promise.race([promise, timeoutPromise]).finally(() => {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+  });
+}
+
+export function initializeTelemetry(env: NodeJS.ProcessEnv = process.env): TelemetryController {
+  if (activeSdk || pendingSdk) {
+    return makeController();
+  }
+
+  const config = getTelemetryConfig(env);
+  if (!config.enabled || isBunRuntime()) {
+    status = 'disabled';
+    return makeController();
+  }
+
+  try {
+    const sdk = createSdk(config);
+    const result = startSdk(sdk);
+    if (result) {
+      pendingSdk = sdk;
+      status = 'starting';
+      const pendingStart = result
+        .then(() => {
+          if (pendingSdk === sdk) {
+            pendingSdk = undefined;
+            activeSdk = sdk;
+            status = 'started';
+            registerShutdownHandlers();
+          }
+        })
+        .catch((error) => {
+          if (pendingSdk === sdk) {
+            pendingSdk = undefined;
+            status = 'failed';
+            emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);
+          }
+        });
+      startPromise = pendingStart;
+      void pendingStart.finally(() => {
+        if (startPromise === pendingStart) {
+          startPromise = undefined;
+        }
+      });
+      return makeController();
+    }
+
+    activeSdk = sdk;
+    status = 'started';
+    registerShutdownHandlers();
+    return makeController();
+  } catch (error) {
+    status = 'failed';
+    emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);
+    return makeController();
+  }
+}
+
+async function performShutdownTelemetry(): Promise<void> {
+  if (startPromise) {
+    await startPromise;
+  }
+
+  if (!activeSdk) {
+    status = status === 'started' ? 'stopped' : status;
+    return;
+  }
+
+  const sdk = activeSdk;
+  try {
+    await sdk.shutdown();
+    activeSdk = undefined;
+    status = 'stopped';
+    unregisterShutdownHandlers();
+  } catch (error) {
+    status = 'started';
+    throw error;
+  }
+}
+
+export function shutdownTelemetry(): Promise<void> {
+  if (!shutdownPromise) {
+    shutdownPromise = performShutdownTelemetry().finally(() => {
+      shutdownPromise = undefined;
+    });
+  }
+
+  return shutdownPromise;
+}
+
+export async function resetTelemetryForTests(): Promise<void> {
+  try {
+    if (startPromise) {
+      await startPromise.catch(() => undefined);
+    }
+
+    if (shutdownPromise) {
+      await shutdownPromise.catch(() => undefined);
+    } else if (activeSdk) {
+      await Promise.resolve(activeSdk.shutdown()).catch(() => undefined);
+    }
+  } finally {
+    activeSdk = undefined;
+    pendingSdk = undefined;
+    startPromise = undefined;
+    shutdownPromise = undefined;
+    status = 'stopped';
+    requestSpans = new WeakMap<IncomingMessage, Span>();
+    unregisterShutdownHandlers();
+  }
+}