📈 feat: Add Prometheus Metrics Endpoint + AWS Credential Providers (#13111)

* feat: add prometheus metrics endpoint * fix: format metrics route spec * chore: update dependencies in package.json and package-lock.json correctly - Bump `@smithy/core` to version 3.24.1 - Update `@aws-sdk/credential-providers` to version 3.1045.0 - Reintroduce `prom-client` dependency in package.json - Remove unnecessary dependencies from package.json * chore: import order * fix: declare s3 presigner peer dependency * fix: normalize shared link metrics path * fix: bound metrics path labels * fix: tighten metrics auth and peers * fix: collapse partial metrics paths
2026-08-31 17:03:24 +00:00 · 2026-05-13 16:49:25 -04:00 · 2026-05-13 16:49:25 -04:00 · 34dd8d5f2a
commit 34dd8d5f2a
parent 83ea3efbc1
8 changed files with 549 additions and 50 deletions
--- a/api/server/index.js
+++ b/api/server/index.js
@ -13,6 +13,7 @@ const { logger, runAsSystem } = require('@librechat/data-schemas');
 const {
  isEnabled,
  apiNotFound,
+  createMetrics,
  ErrorController,
  memoryDiagnostics,
  performStartupChecks,
@ -20,8 +21,8 @@ const {
  GenerationJobManager,
  createStreamServices,
  initializeFileStorage,
-  updateInterfacePermissions,
  preAuthTenantMiddleware,
+  updateInterfacePermissions,
 } = require('@librechat/api');
 const { connectDb, indexSync } = require('~/db');
 const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
@ -53,6 +54,11 @@ const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default
 const app = express();

 const startServer = async () => {
+  const { metricsMiddleware, metricsRouter } = createMetrics();
+  if (!process.env.METRICS_SECRET) {
+    logger.warn('[metrics] METRICS_SECRET is not set - /metrics will return 401 for all requests');
+  }
+
  if (typeof Bun !== 'undefined') {
    axios.defaults.headers.common['Accept-Encoding'] = 'gzip';
  }
@ -107,6 +113,7 @@ const startServer = async () => {
  app.get('/health', (_req, res) => res.status(200).send('OK'));

  /* Middleware */
+  app.use(metricsMiddleware);
  app.use(noIndex);
  app.use(express.json({ limit: '3mb' }));
  app.use(express.urlencoded({ extended: true, limit: '3mb' }));
@ -199,6 +206,8 @@ const startServer = async () => {
  app.use('/api/tags', routes.tags);
  app.use('/api/mcp', routes.mcp);

+  app.use('/metrics', metricsRouter);
+
  /** 404 for unmatched API routes */
  app.use('/api', apiNotFound);

--- a/api/server/index.metrics.spec.js
+++ b/api/server/index.metrics.spec.js
@ -0,0 +1,164 @@
+const fs = require('fs');
+const request = require('supertest');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+const mongoose = require('mongoose');
+
+jest.mock('~/server/services/Config', () => ({
+  loadCustomConfig: jest.fn(() => Promise.resolve({})),
+  getAppConfig: jest.fn().mockResolvedValue({
+    paths: {
+      uploads: '/tmp',
+      dist: '/tmp/dist',
+      fonts: '/tmp/fonts',
+      assets: '/tmp/assets',
+    },
+    fileStrategy: 'local',
+    imageOutputType: 'PNG',
+  }),
+  setCachedTools: jest.fn(),
+}));
+
+jest.mock('~/app/clients/tools', () => ({
+  createOpenAIImageTools: jest.fn(() => []),
+  createYouTubeTools: jest.fn(() => []),
+  manifestToolMap: {},
+  toolkits: [],
+}));
+
+jest.mock('~/config', () => ({
+  createMCPServersRegistry: jest.fn(),
+  createMCPManager: jest.fn().mockResolvedValue({
+    getAppToolFunctions: jest.fn().mockResolvedValue({}),
+  }),
+}));
+
+describe('Server metrics route', () => {
+  jest.setTimeout(30_000);
+
+  let mongoServer;
+  let app;
+
+  const originalReadFileSync = fs.readFileSync;
+
+  beforeAll(() => {
+    fs.readFileSync = function (filepath, options) {
+      if (filepath.includes('index.html')) {
+        return '<!DOCTYPE html><html><head><title>LibreChat</title></head><body><div id="root"></div></body></html>';
+      }
+      return originalReadFileSync(filepath, options);
+    };
+  });
+
+  afterAll(() => {
+    fs.readFileSync = originalReadFileSync;
+  });
+
+  beforeAll(async () => {
+    const fs = require('fs');
+    const path = require('path');
+
+    const dirs = ['/tmp/dist', '/tmp/fonts', '/tmp/assets'];
+    dirs.forEach((dir) => {
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+    });
+
+    fs.writeFileSync(
+      path.join('/tmp/dist', 'index.html'),
+      '<!DOCTYPE html><html><head><title>LibreChat</title></head><body><div id="root"></div></body></html>',
+    );
+
+    mongoServer = await MongoMemoryServer.create();
+    process.env.MONGO_URI = mongoServer.getUri();
+    process.env.PORT = '0';
+    app = require('~/server');
+
+    await healthCheckPoll(app);
+  });
+
+  afterEach(() => {
+    delete process.env.METRICS_SECRET;
+  });
+
+  afterAll(async () => {
+    await mongoServer.stop();
+    await mongoose.disconnect();
+  });
+
+  it('returns 401 at /metrics when METRICS_SECRET is unset', async () => {
+    const response = await request(app).get('/metrics');
+    expect(response.status).toBe(401);
+  });
+
+  it('returns 401 at /metrics when no token provided', async () => {
+    process.env.METRICS_SECRET = 'test-secret';
+
+    const response = await request(app).get('/metrics');
+
+    expect(response.status).toBe(401);
+  });
+
+  it('returns 401 at /metrics when wrong token provided', async () => {
+    process.env.METRICS_SECRET = 'test-secret';
+
+    const response = await request(app).get('/metrics').set('Authorization', 'Bearer wrong-token');
+
+    expect(response.status).toBe(401);
+  });
+
+  it('returns 401 at /metrics when the bearer scheme is omitted', async () => {
+    process.env.METRICS_SECRET = 'test-secret';
+
+    const response = await request(app).get('/metrics').set('Authorization', 'test-secret');
+
+    expect(response.status).toBe(401);
+  });
+
+  it('returns 401 at /metrics for non-bearer auth schemes', async () => {
+    process.env.METRICS_SECRET = 'test-secret';
+
+    const response = await request(app).get('/metrics').set('Authorization', 'Basic test-secret');
+
+    expect(response.status).toBe(401);
+  });
+
+  it('exposes Prometheus metrics at /metrics with correct bearer token', async () => {
+    process.env.METRICS_SECRET = 'test-secret';
+
+    const response = await request(app).get('/metrics').set('Authorization', 'Bearer test-secret');
+
+    expect(response.status).toBe(200);
+    expect(response.headers['content-type']).toMatch(/text\/plain/);
+    expect(response.text).toMatch(/^# HELP /m);
+    expect(response.text).toMatch(/^# TYPE /m);
+  });
+
+  it('accepts lowercase bearer scheme at /metrics', async () => {
+    process.env.METRICS_SECRET = 'test-secret';
+
+    const response = await request(app).get('/metrics').set('Authorization', 'bearer test-secret');
+
+    expect(response.status).toBe(200);
+  });
+});
+
+async function healthCheckPoll(app, retries = 0) {
+  const maxRetries = Math.floor(10000 / 30);
+  try {
+    const response = await request(app).get('/health');
+    if (response.status === 200) {
+      return;
+    }
+  } catch {
+    // Ignore connection errors during polling.
+  }
+
+  if (retries < maxRetries) {
+    await new Promise((resolve) => setTimeout(resolve, 30));
+    await healthCheckPoll(app, retries + 1);
+    return;
+  }
+
+  throw new Error('App did not become healthy within 10 seconds.');
+}