📡 feat: Add Backend OpenTelemetry Tracing (#12909)

* feat: add backend OpenTelemetry tracing

* fix: address telemetry type checks

* fix: mark aborted telemetry requests as errors

* fix: record telemetry identity after auth

* fix: avoid forced telemetry signal exit

* fix: harden telemetry request attribution

* fix: record telemetry errors on request span

* chore: order imports and reorganize middleware usage

* fix: reduce telemetry startup overhead

* fix: preserve live telemetry controller state

* fix: redact telemetry URL attributes

api/server/index.js

@@ -1,4 +1,4 @@
-require('dotenv').config();
+const telemetry = require('./telemetry');
 const fs = require('fs');
 const path = require('path');
 require('module-alias')({ base: path.resolve(__dirname, '..') });
@@ -25,22 +25,22 @@ const {
   updateInterfacePermissions,
 } = require('@librechat/api');
 const { connectDb, indexSync } = require('~/db');
-const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
 const {
-  getRoleByName,
   updateAccessPermissions,
-  seedDatabase,
   sweepOrphanedPreviews,
+  getRoleByName,
+  seedDatabase,
 } = require('~/models');
+const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
 const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');
 const createValidateImageRequest = require('./middleware/validateImageRequest');
 const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
 const { checkMigrations } = require('./services/start/migration');
+const optionalJwtAuth = require('./middleware/optionalJwtAuth');
 const initializeMCPs = require('./services/initializeMCPs');
 const configureSocialLogins = require('./socialLogins');
 const { getAppConfig } = require('./services/Config');
 const staticCache = require('./utils/staticCache');
-const optionalJwtAuth = require('./middleware/optionalJwtAuth');
 const noIndex = require('./middleware/noIndex');
 const routes = require('./routes');
 
@@ -146,6 +146,10 @@ const startServer = async () => {
   app.use(staticCache(appConfig.paths.fonts));
   app.use(staticCache(appConfig.paths.assets));
 
+  if (telemetry.enabled) {
+    app.use(telemetry.telemetryMiddleware);
+  }
+
   if (!ALLOW_SOCIAL_LOGIN) {
     console.warn('Social logins are disabled. Set ALLOW_SOCIAL_LOGIN=true to enable them.');
   }
@@ -227,6 +231,10 @@ const startServer = async () => {
     res.send(updatedIndexHtml);
   });
 
+  /** Record trace errors before the final error controller. */
+  if (telemetry.enabled) {
+    app.use(telemetry.telemetryErrorMiddleware);
+  }
   /** Error handler (must be last - Express identifies error middleware by its 4-arg signature) */
   app.use(ErrorController);
 

api/server/index.spec.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const path = require('path');
 const request = require('supertest');
 const { MongoMemoryServer } = require('mongodb-memory-server');
 const mongoose = require('mongoose');
@@ -32,6 +33,56 @@ jest.mock('~/config', () => ({
   }),
 }));
 
+jest.mock(
+  '@librechat/api/telemetry',
+  () => ({
+    initializeTelemetry: jest.fn(() => ({
+      enabled: false,
+      status: 'disabled',
+      shutdown: jest.fn(),
+    })),
+    telemetryMiddleware: jest.fn((_req, _res, next) => next()),
+    telemetryErrorMiddleware: jest.fn((err, _req, _res, next) => next(err)),
+  }),
+  { virtual: true },
+);
+
+describe('Telemetry wiring', () => {
+  const source = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8');
+
+  it('loads telemetry before other server imports', () => {
+    const firstStatement = source
+      .split('\n')
+      .map((line) => line.trim())
+      .find(Boolean);
+
+    expect(firstStatement).toBe("const telemetry = require('./telemetry');");
+  });
+
+  it('mounts telemetry middleware after static assets and before routes', () => {
+    const telemetryMiddlewareIndex = source.indexOf('app.use(telemetry.telemetryMiddleware);');
+    const staticAssetsIndex = source.indexOf('app.use(staticCache(appConfig.paths.assets));');
+    const apiRoutesIndex = source.indexOf("app.use('/api/auth'");
+
+    expect(telemetryMiddlewareIndex).toBeGreaterThan(-1);
+    expect(staticAssetsIndex).toBeGreaterThan(-1);
+    expect(apiRoutesIndex).toBeGreaterThan(-1);
+    expect(staticAssetsIndex).toBeLessThan(telemetryMiddlewareIndex);
+    expect(telemetryMiddlewareIndex).toBeLessThan(apiRoutesIndex);
+  });
+
+  it('mounts telemetry error middleware before ErrorController', () => {
+    const telemetryErrorMiddlewareIndex = source.indexOf(
+      'app.use(telemetry.telemetryErrorMiddleware);',
+    );
+    const errorControllerIndex = source.indexOf('app.use(ErrorController);');
+
+    expect(telemetryErrorMiddlewareIndex).toBeGreaterThan(-1);
+    expect(errorControllerIndex).toBeGreaterThan(-1);
+    expect(telemetryErrorMiddlewareIndex).toBeLessThan(errorControllerIndex);
+  });
+});
+
 describe('Server Configuration', () => {
   // Increase the default timeout to allow for Mongo cleanup
   jest.setTimeout(30_000);

api/server/telemetry.js

@@ -0,0 +1,40 @@
+require('dotenv').config();
+
+function isTruthy(value) {
+  return value?.trim().toLowerCase() === 'true';
+}
+
+function isTelemetryEnabled() {
+  return isTruthy(process.env.OTEL_TRACING_ENABLED) && !isTruthy(process.env.OTEL_SDK_DISABLED);
+}
+
+if (isTelemetryEnabled()) {
+  const {
+    initializeTelemetry,
+    telemetryMiddleware,
+    telemetryErrorMiddleware,
+  } = require('@librechat/api/telemetry');
+  const controller = initializeTelemetry();
+
+  module.exports = {
+    get enabled() {
+      return controller.enabled;
+    },
+    get status() {
+      return controller.status;
+    },
+    shutdown: controller.shutdown,
+    telemetryMiddleware,
+    telemetryErrorMiddleware,
+  };
+} else {
+  module.exports = {
+    enabled: false,
+    get status() {
+      return 'disabled';
+    },
+    shutdown: async () => {},
+    telemetryMiddleware: (_req, _res, next) => next(),
+    telemetryErrorMiddleware: (err, _req, _res, next) => next(err),
+  };
+}

api/server/telemetry.spec.js

@@ -0,0 +1,91 @@
+describe('telemetry bootstrap', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    jest.resetModules();
+    process.env = { ...originalEnv };
+    delete process.env.OTEL_SDK_DISABLED;
+    delete process.env.OTEL_TRACING_ENABLED;
+    jest.doMock('dotenv', () => ({
+      config: jest.fn(),
+    }));
+  });
+
+  afterEach(() => {
+    process.env = originalEnv;
+    jest.dontMock('dotenv');
+    jest.dontMock('@librechat/api/telemetry');
+    jest.resetModules();
+  });
+
+  it('does not load OpenTelemetry packages by default', () => {
+    jest.doMock(
+      '@librechat/api/telemetry',
+      () => {
+        throw new Error('telemetry package should not load when tracing is disabled');
+      },
+      { virtual: true },
+    );
+
+    const telemetry = require('./telemetry');
+
+    expect(telemetry.enabled).toBe(false);
+    expect(telemetry.status).toBe('disabled');
+  });
+
+  it('does not load OpenTelemetry packages when the SDK is disabled', () => {
+    process.env.OTEL_SDK_DISABLED = 'true';
+    process.env.OTEL_TRACING_ENABLED = 'true';
+    jest.doMock(
+      '@librechat/api/telemetry',
+      () => {
+        throw new Error('telemetry package should not load when the SDK is disabled');
+      },
+      { virtual: true },
+    );
+
+    const telemetry = require('./telemetry');
+
+    expect(telemetry.enabled).toBe(false);
+    expect(telemetry.status).toBe('disabled');
+  });
+
+  it('loads and exposes telemetry middleware when tracing is enabled', () => {
+    process.env.OTEL_TRACING_ENABLED = 'true';
+    let enabled = true;
+    let status = 'starting';
+    const telemetryMiddleware = jest.fn();
+    const telemetryErrorMiddleware = jest.fn();
+    const initializeTelemetry = jest.fn(() => ({
+      get enabled() {
+        return enabled;
+      },
+      get status() {
+        return status;
+      },
+      shutdown: jest.fn(),
+    }));
+    jest.doMock(
+      '@librechat/api/telemetry',
+      () => ({
+        initializeTelemetry,
+        telemetryMiddleware,
+        telemetryErrorMiddleware,
+      }),
+      { virtual: true },
+    );
+
+    const telemetry = require('./telemetry');
+
+    expect(initializeTelemetry).toHaveBeenCalledTimes(1);
+    expect(telemetry.enabled).toBe(true);
+    expect(telemetry.status).toBe('starting');
+    expect(telemetry.telemetryMiddleware).toBe(telemetryMiddleware);
+    expect(telemetry.telemetryErrorMiddleware).toBe(telemetryErrorMiddleware);
+
+    enabled = false;
+    status = 'failed';
+    expect(telemetry.enabled).toBe(false);
+    expect(telemetry.status).toBe('failed');
+  });
+});