mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
📡 feat: Add Backend OpenTelemetry Tracing (#12909)
* feat: add backend OpenTelemetry tracing * fix: address telemetry type checks * fix: mark aborted telemetry requests as errors * fix: record telemetry identity after auth * fix: avoid forced telemetry signal exit * fix: harden telemetry request attribution * fix: record telemetry errors on request span * chore: order imports and reorganize middleware usage * fix: reduce telemetry startup overhead * fix: preserve live telemetry controller state * fix: redact telemetry URL attributes
This commit is contained in:
Danny Avila
GitHub
parent
7e4c5d9ded
commit
050b7fd43a
16 changed files with 1864 additions and 8 deletions
17
.env.example
17
.env.example
|
|
@ -110,6 +110,23 @@ NODE_MAX_OLD_SPACE_SIZE=6144
|
|||
# LANGFUSE_SECRET_KEY=
|
||||
# LANGFUSE_BASE_URL=
|
||||
|
||||
#=======================#
|
||||
# OpenTelemetry Tracing #
|
||||
#=======================#
|
||||
|
||||
# Enables backend OpenTelemetry tracing. General backend visibility only;
|
||||
# use Langfuse for GenAI-specific prompt/model observability.
|
||||
# OTEL_TRACING_ENABLED=false
|
||||
# OTEL_SERVICE_NAME=librechat
|
||||
# OTEL_SERVICE_VERSION=
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=
|
||||
# OTEL_TRACES_EXPORTER=otlp
|
||||
# OTEL_TRACES_SAMPLER=parentbased_always_on
|
||||
# OTEL_LOG_LEVEL=INFO
|
||||
# OTEL_SDK_DISABLED=false
|
||||
|
||||
#===================================================#
|
||||
# Endpoints #
|
||||
#===================================================#
|
||||
|
|
|
|||
|
|
@ -52,6 +52,16 @@
|
|||
"@microsoft/microsoft-graph-client": "^3.0.7",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@node-saml/passport-saml": "^5.1.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/instrumentation-express": "^0.56.0",
|
||||
"@opentelemetry/instrumentation-http": "^0.207.0",
|
||||
"@opentelemetry/instrumentation-ioredis": "^0.55.0",
|
||||
"@opentelemetry/instrumentation-mongodb": "^0.60.0",
|
||||
"@opentelemetry/instrumentation-mongoose": "^0.54.0",
|
||||
"@opentelemetry/instrumentation-undici": "^0.18.0",
|
||||
"@opentelemetry/resources": "^2.6.1",
|
||||
"@opentelemetry/sdk-node": "^0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.39.0",
|
||||
"@smithy/node-http-handler": "^4.4.5",
|
||||
"ai-tokenizer": "^1.0.6",
|
||||
"axios": "^1.16.0",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
require('dotenv').config();
|
||||
const telemetry = require('./telemetry');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
require('module-alias')({ base: path.resolve(__dirname, '..') });
|
||||
|
|
@ -25,22 +25,22 @@ const {
|
|||
updateInterfacePermissions,
|
||||
} = require('@librechat/api');
|
||||
const { connectDb, indexSync } = require('~/db');
|
||||
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
|
||||
const {
|
||||
getRoleByName,
|
||||
updateAccessPermissions,
|
||||
seedDatabase,
|
||||
sweepOrphanedPreviews,
|
||||
getRoleByName,
|
||||
seedDatabase,
|
||||
} = require('~/models');
|
||||
const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
|
||||
const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');
|
||||
const createValidateImageRequest = require('./middleware/validateImageRequest');
|
||||
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
|
||||
const { checkMigrations } = require('./services/start/migration');
|
||||
const optionalJwtAuth = require('./middleware/optionalJwtAuth');
|
||||
const initializeMCPs = require('./services/initializeMCPs');
|
||||
const configureSocialLogins = require('./socialLogins');
|
||||
const { getAppConfig } = require('./services/Config');
|
||||
const staticCache = require('./utils/staticCache');
|
||||
const optionalJwtAuth = require('./middleware/optionalJwtAuth');
|
||||
const noIndex = require('./middleware/noIndex');
|
||||
const routes = require('./routes');
|
||||
|
||||
|
|
@ -146,6 +146,10 @@ const startServer = async () => {
|
|||
app.use(staticCache(appConfig.paths.fonts));
|
||||
app.use(staticCache(appConfig.paths.assets));
|
||||
|
||||
if (telemetry.enabled) {
|
||||
app.use(telemetry.telemetryMiddleware);
|
||||
}
|
||||
|
||||
if (!ALLOW_SOCIAL_LOGIN) {
|
||||
console.warn('Social logins are disabled. Set ALLOW_SOCIAL_LOGIN=true to enable them.');
|
||||
}
|
||||
|
|
@ -227,6 +231,10 @@ const startServer = async () => {
|
|||
res.send(updatedIndexHtml);
|
||||
});
|
||||
|
||||
/** Record trace errors before the final error controller. */
|
||||
if (telemetry.enabled) {
|
||||
app.use(telemetry.telemetryErrorMiddleware);
|
||||
}
|
||||
/** Error handler (must be last - Express identifies error middleware by its 4-arg signature) */
|
||||
app.use(ErrorController);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const request = require('supertest');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
const mongoose = require('mongoose');
|
||||
|
|
@ -32,6 +33,56 @@ jest.mock('~/config', () => ({
|
|||
}),
|
||||
}));
|
||||
|
||||
jest.mock(
|
||||
'@librechat/api/telemetry',
|
||||
() => ({
|
||||
initializeTelemetry: jest.fn(() => ({
|
||||
enabled: false,
|
||||
status: 'disabled',
|
||||
shutdown: jest.fn(),
|
||||
})),
|
||||
telemetryMiddleware: jest.fn((_req, _res, next) => next()),
|
||||
telemetryErrorMiddleware: jest.fn((err, _req, _res, next) => next(err)),
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
describe('Telemetry wiring', () => {
|
||||
const source = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8');
|
||||
|
||||
it('loads telemetry before other server imports', () => {
|
||||
const firstStatement = source
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.find(Boolean);
|
||||
|
||||
expect(firstStatement).toBe("const telemetry = require('./telemetry');");
|
||||
});
|
||||
|
||||
it('mounts telemetry middleware after static assets and before routes', () => {
|
||||
const telemetryMiddlewareIndex = source.indexOf('app.use(telemetry.telemetryMiddleware);');
|
||||
const staticAssetsIndex = source.indexOf('app.use(staticCache(appConfig.paths.assets));');
|
||||
const apiRoutesIndex = source.indexOf("app.use('/api/auth'");
|
||||
|
||||
expect(telemetryMiddlewareIndex).toBeGreaterThan(-1);
|
||||
expect(staticAssetsIndex).toBeGreaterThan(-1);
|
||||
expect(apiRoutesIndex).toBeGreaterThan(-1);
|
||||
expect(staticAssetsIndex).toBeLessThan(telemetryMiddlewareIndex);
|
||||
expect(telemetryMiddlewareIndex).toBeLessThan(apiRoutesIndex);
|
||||
});
|
||||
|
||||
it('mounts telemetry error middleware before ErrorController', () => {
|
||||
const telemetryErrorMiddlewareIndex = source.indexOf(
|
||||
'app.use(telemetry.telemetryErrorMiddleware);',
|
||||
);
|
||||
const errorControllerIndex = source.indexOf('app.use(ErrorController);');
|
||||
|
||||
expect(telemetryErrorMiddlewareIndex).toBeGreaterThan(-1);
|
||||
expect(errorControllerIndex).toBeGreaterThan(-1);
|
||||
expect(telemetryErrorMiddlewareIndex).toBeLessThan(errorControllerIndex);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Server Configuration', () => {
|
||||
// Increase the default timeout to allow for Mongo cleanup
|
||||
jest.setTimeout(30_000);
|
||||
|
|
|
|||
40
api/server/telemetry.js
Normal file
40
api/server/telemetry.js
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
require('dotenv').config();
|
||||
|
||||
function isTruthy(value) {
|
||||
return value?.trim().toLowerCase() === 'true';
|
||||
}
|
||||
|
||||
function isTelemetryEnabled() {
|
||||
return isTruthy(process.env.OTEL_TRACING_ENABLED) && !isTruthy(process.env.OTEL_SDK_DISABLED);
|
||||
}
|
||||
|
||||
if (isTelemetryEnabled()) {
|
||||
const {
|
||||
initializeTelemetry,
|
||||
telemetryMiddleware,
|
||||
telemetryErrorMiddleware,
|
||||
} = require('@librechat/api/telemetry');
|
||||
const controller = initializeTelemetry();
|
||||
|
||||
module.exports = {
|
||||
get enabled() {
|
||||
return controller.enabled;
|
||||
},
|
||||
get status() {
|
||||
return controller.status;
|
||||
},
|
||||
shutdown: controller.shutdown,
|
||||
telemetryMiddleware,
|
||||
telemetryErrorMiddleware,
|
||||
};
|
||||
} else {
|
||||
module.exports = {
|
||||
enabled: false,
|
||||
get status() {
|
||||
return 'disabled';
|
||||
},
|
||||
shutdown: async () => {},
|
||||
telemetryMiddleware: (_req, _res, next) => next(),
|
||||
telemetryErrorMiddleware: (err, _req, _res, next) => next(err),
|
||||
};
|
||||
}
|
||||
91
api/server/telemetry.spec.js
Normal file
91
api/server/telemetry.spec.js
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
describe('telemetry bootstrap', () => {
|
||||
const originalEnv = process.env;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetModules();
|
||||
process.env = { ...originalEnv };
|
||||
delete process.env.OTEL_SDK_DISABLED;
|
||||
delete process.env.OTEL_TRACING_ENABLED;
|
||||
jest.doMock('dotenv', () => ({
|
||||
config: jest.fn(),
|
||||
}));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
jest.dontMock('dotenv');
|
||||
jest.dontMock('@librechat/api/telemetry');
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
it('does not load OpenTelemetry packages by default', () => {
|
||||
jest.doMock(
|
||||
'@librechat/api/telemetry',
|
||||
() => {
|
||||
throw new Error('telemetry package should not load when tracing is disabled');
|
||||
},
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
const telemetry = require('./telemetry');
|
||||
|
||||
expect(telemetry.enabled).toBe(false);
|
||||
expect(telemetry.status).toBe('disabled');
|
||||
});
|
||||
|
||||
it('does not load OpenTelemetry packages when the SDK is disabled', () => {
|
||||
process.env.OTEL_SDK_DISABLED = 'true';
|
||||
process.env.OTEL_TRACING_ENABLED = 'true';
|
||||
jest.doMock(
|
||||
'@librechat/api/telemetry',
|
||||
() => {
|
||||
throw new Error('telemetry package should not load when the SDK is disabled');
|
||||
},
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
const telemetry = require('./telemetry');
|
||||
|
||||
expect(telemetry.enabled).toBe(false);
|
||||
expect(telemetry.status).toBe('disabled');
|
||||
});
|
||||
|
||||
it('loads and exposes telemetry middleware when tracing is enabled', () => {
|
||||
process.env.OTEL_TRACING_ENABLED = 'true';
|
||||
let enabled = true;
|
||||
let status = 'starting';
|
||||
const telemetryMiddleware = jest.fn();
|
||||
const telemetryErrorMiddleware = jest.fn();
|
||||
const initializeTelemetry = jest.fn(() => ({
|
||||
get enabled() {
|
||||
return enabled;
|
||||
},
|
||||
get status() {
|
||||
return status;
|
||||
},
|
||||
shutdown: jest.fn(),
|
||||
}));
|
||||
jest.doMock(
|
||||
'@librechat/api/telemetry',
|
||||
() => ({
|
||||
initializeTelemetry,
|
||||
telemetryMiddleware,
|
||||
telemetryErrorMiddleware,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
const telemetry = require('./telemetry');
|
||||
|
||||
expect(initializeTelemetry).toHaveBeenCalledTimes(1);
|
||||
expect(telemetry.enabled).toBe(true);
|
||||
expect(telemetry.status).toBe('starting');
|
||||
expect(telemetry.telemetryMiddleware).toBe(telemetryMiddleware);
|
||||
expect(telemetry.telemetryErrorMiddleware).toBe(telemetryErrorMiddleware);
|
||||
|
||||
enabled = false;
|
||||
status = 'failed';
|
||||
expect(telemetry.enabled).toBe(false);
|
||||
expect(telemetry.status).toBe('failed');
|
||||
});
|
||||
});
|
||||
281
package-lock.json
generated
281
package-lock.json
generated
|
|
@ -67,6 +67,16 @@
|
|||
"@microsoft/microsoft-graph-client": "^3.0.7",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@node-saml/passport-saml": "^5.1.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/instrumentation-express": "^0.56.0",
|
||||
"@opentelemetry/instrumentation-http": "^0.207.0",
|
||||
"@opentelemetry/instrumentation-ioredis": "^0.55.0",
|
||||
"@opentelemetry/instrumentation-mongodb": "^0.60.0",
|
||||
"@opentelemetry/instrumentation-mongoose": "^0.54.0",
|
||||
"@opentelemetry/instrumentation-undici": "^0.18.0",
|
||||
"@opentelemetry/resources": "^2.6.1",
|
||||
"@opentelemetry/sdk-node": "^0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.39.0",
|
||||
"@smithy/node-http-handler": "^4.4.5",
|
||||
"ai-tokenizer": "^1.0.6",
|
||||
"axios": "^1.16.0",
|
||||
|
|
@ -12866,7 +12876,6 @@
|
|||
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.1.tgz",
|
||||
"integrity": "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g==",
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
|
|
@ -14245,6 +14254,120 @@
|
|||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-express": {
|
||||
"version": "0.56.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.56.0.tgz",
|
||||
"integrity": "sha512-rMV0WUTtAGEhHrHl3uDRIO97EkNUp4ewrW2iRVuP7kaV5qRT2b1pPV5PE75oR3GyLLSTooSAzGWl6CTm8eftKQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "^2.0.0",
|
||||
"@opentelemetry/instrumentation": "^0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.27.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-http": {
|
||||
"version": "0.207.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.207.0.tgz",
|
||||
"integrity": "sha512-FC4i5hVixTzuhg4SV2ycTEAYx+0E2hm+GwbdoVPSA6kna0pPVI4etzaA9UkpJ9ussumQheFXP6rkGIaFJjMxsw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "2.2.0",
|
||||
"@opentelemetry/instrumentation": "0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0",
|
||||
"forwarded-parse": "2.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz",
|
||||
"integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-ioredis": {
|
||||
"version": "0.55.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.55.0.tgz",
|
||||
"integrity": "sha512-ASuBMzh0ImmfOnWj9vCPtBMqSjr54/r/HluUIylwZB7xzTU6gL2SfybxySJMzEL9+386gJJVApwQktVznAtrWA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/instrumentation": "^0.207.0",
|
||||
"@opentelemetry/redis-common": "^0.38.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-mongodb": {
|
||||
"version": "0.60.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.60.0.tgz",
|
||||
"integrity": "sha512-8mKW2oyyWdYOKYpu70AGGAvLnExGlOoFT+Ylk9hJvWYHR5f6IrmVqwUMlzPM2WC4ihQ9crvqrtldMPmlVezTqg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/instrumentation": "^0.207.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-mongoose": {
|
||||
"version": "0.54.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.54.0.tgz",
|
||||
"integrity": "sha512-R2JzMrxiz3R9m+cO6iVC85bma3E3vdBr06F+D4zGcEkJS9FaCRw6+Hdb+aQ1AaDRwEMTaw+F28+6VdMpniBOXQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "^2.0.0",
|
||||
"@opentelemetry/instrumentation": "^0.207.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation-undici": {
|
||||
"version": "0.18.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.18.0.tgz",
|
||||
"integrity": "sha512-NalxLuZV621Xq4IhQkC+OXoZjAT8Xf6vYRdTjHitOXMU+4l/peRY05V7wGr4d7huf+vjyQry0XKlyhsEr4ouNw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "^2.0.0",
|
||||
"@opentelemetry/instrumentation": "^0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.24.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/instrumentation/node_modules/@opentelemetry/api-logs": {
|
||||
"version": "0.207.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.207.0.tgz",
|
||||
|
|
@ -14504,12 +14627,20 @@
|
|||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/redis-common": {
|
||||
"version": "0.38.3",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz",
|
||||
"integrity": "sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": "^18.19.0 || >=20.6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/resources": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.6.1.tgz",
|
||||
"integrity": "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA==",
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "2.6.1",
|
||||
"@opentelemetry/semantic-conventions": "^1.29.0"
|
||||
|
|
@ -28134,6 +28265,12 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded-parse": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz",
|
||||
"integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fraction.js": {
|
||||
"version": "5.3.4",
|
||||
"resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz",
|
||||
|
|
@ -28286,6 +28423,67 @@
|
|||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/gcp-metadata": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz",
|
||||
"integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==",
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"gaxios": "^5.0.0",
|
||||
"json-bigint": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/gcp-metadata/node_modules/agent-base": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/gcp-metadata/node_modules/gaxios": {
|
||||
"version": "5.1.3",
|
||||
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz",
|
||||
"integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==",
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"extend": "^3.0.2",
|
||||
"https-proxy-agent": "^5.0.0",
|
||||
"is-stream": "^2.0.0",
|
||||
"node-fetch": "^2.6.9"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/gcp-metadata/node_modules/https-proxy-agent": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
|
||||
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/generic-names": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/generic-names/-/generic-names-4.0.0.tgz",
|
||||
|
|
@ -28585,6 +28783,7 @@
|
|||
"version": "6.1.1",
|
||||
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz",
|
||||
"integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"gaxios": "^6.1.1",
|
||||
"google-logging-utils": "^0.0.2",
|
||||
|
|
@ -28598,6 +28797,7 @@
|
|||
"version": "0.0.2",
|
||||
"resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz",
|
||||
"integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
|
|
@ -34726,6 +34926,52 @@
|
|||
"node": ">=20.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-memory-server-core/node_modules/gaxios": {
|
||||
"version": "7.1.4",
|
||||
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz",
|
||||
"integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"extend": "^3.0.2",
|
||||
"https-proxy-agent": "^7.0.1",
|
||||
"node-fetch": "^3.3.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-memory-server-core/node_modules/gcp-metadata": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz",
|
||||
"integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"gaxios": "^7.0.0",
|
||||
"google-logging-utils": "^1.0.0",
|
||||
"json-bigint": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-memory-server-core/node_modules/google-logging-utils": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz",
|
||||
"integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-memory-server-core/node_modules/mongodb": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.1.1.tgz",
|
||||
|
|
@ -34787,6 +35033,27 @@
|
|||
"node": ">=20.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-memory-server-core/node_modules/node-fetch": {
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
|
||||
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-fetch"
|
||||
}
|
||||
},
|
||||
"node_modules/mongoose": {
|
||||
"version": "8.23.1",
|
||||
"resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.23.1.tgz",
|
||||
|
|
@ -44692,6 +44959,16 @@
|
|||
"@librechat/agents": "^3.1.86",
|
||||
"@librechat/data-schemas": "*",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/instrumentation-express": "^0.56.0",
|
||||
"@opentelemetry/instrumentation-http": "^0.207.0",
|
||||
"@opentelemetry/instrumentation-ioredis": "^0.55.0",
|
||||
"@opentelemetry/instrumentation-mongodb": "^0.60.0",
|
||||
"@opentelemetry/instrumentation-mongoose": "^0.54.0",
|
||||
"@opentelemetry/instrumentation-undici": "^0.18.0",
|
||||
"@opentelemetry/resources": "^2.6.1",
|
||||
"@opentelemetry/sdk-node": "^0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.39.0",
|
||||
"@smithy/node-http-handler": "^4.4.5",
|
||||
"ai-tokenizer": "^1.0.6",
|
||||
"axios": "^1.16.0",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@
|
|||
".": {
|
||||
"require": "./dist/index.js",
|
||||
"types": "./dist/types/index.d.ts"
|
||||
},
|
||||
"./telemetry": {
|
||||
"require": "./dist/telemetry.js",
|
||||
"types": "./dist/types/telemetry/index.d.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
|
|
@ -103,6 +107,16 @@
|
|||
"@librechat/agents": "^3.1.86",
|
||||
"@librechat/data-schemas": "*",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/instrumentation-express": "^0.56.0",
|
||||
"@opentelemetry/instrumentation-http": "^0.207.0",
|
||||
"@opentelemetry/instrumentation-ioredis": "^0.55.0",
|
||||
"@opentelemetry/instrumentation-mongodb": "^0.60.0",
|
||||
"@opentelemetry/instrumentation-mongoose": "^0.54.0",
|
||||
"@opentelemetry/instrumentation-undici": "^0.18.0",
|
||||
"@opentelemetry/resources": "^2.6.1",
|
||||
"@opentelemetry/sdk-node": "^0.207.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.39.0",
|
||||
"@smithy/node-http-handler": "^4.4.5",
|
||||
"ai-tokenizer": "^1.0.6",
|
||||
"axios": "^1.16.0",
|
||||
|
|
|
|||
|
|
@ -45,7 +45,10 @@ const plugins = [
|
|||
];
|
||||
|
||||
const cjsBuild = {
|
||||
input: 'src/index.ts',
|
||||
input: {
|
||||
index: 'src/index.ts',
|
||||
telemetry: 'src/telemetry/index.ts',
|
||||
},
|
||||
output: {
|
||||
dir: 'dist',
|
||||
format: 'cjs',
|
||||
|
|
|
|||
46
packages/api/src/telemetry/config.spec.ts
Normal file
46
packages/api/src/telemetry/config.spec.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import { getTelemetryConfig } from './config';
|
||||
|
||||
describe('getTelemetryConfig', () => {
|
||||
it('defaults tracing off', () => {
|
||||
const config = getTelemetryConfig({});
|
||||
|
||||
expect(config.enabled).toBe(false);
|
||||
expect(config.sdkDisabled).toBe(false);
|
||||
expect(config.serviceName).toBe('librechat');
|
||||
expect(config.healthPath).toBe('/health');
|
||||
});
|
||||
|
||||
it('enables tracing only when OTEL_TRACING_ENABLED is true', () => {
|
||||
expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'true' }).enabled).toBe(true);
|
||||
expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'TRUE' }).enabled).toBe(true);
|
||||
expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'false' }).enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('lets OTEL_SDK_DISABLED override tracing enablement', () => {
|
||||
const config = getTelemetryConfig({
|
||||
OTEL_SDK_DISABLED: 'true',
|
||||
OTEL_TRACING_ENABLED: 'true',
|
||||
});
|
||||
|
||||
expect(config.enabled).toBe(false);
|
||||
expect(config.sdkDisabled).toBe(true);
|
||||
});
|
||||
|
||||
it('uses standard service env vars when provided', () => {
|
||||
const config = getTelemetryConfig({
|
||||
OTEL_SERVICE_NAME: ' librechat-api ',
|
||||
OTEL_SERVICE_VERSION: ' 1.2.3 ',
|
||||
});
|
||||
|
||||
expect(config.serviceName).toBe('librechat-api');
|
||||
expect(config.serviceVersion).toBe('1.2.3');
|
||||
});
|
||||
|
||||
it('falls back to npm package version when service version is absent', () => {
|
||||
const config = getTelemetryConfig({
|
||||
npm_package_version: '0.8.5',
|
||||
});
|
||||
|
||||
expect(config.serviceVersion).toBe('0.8.5');
|
||||
});
|
||||
});
|
||||
43
packages/api/src/telemetry/config.ts
Normal file
43
packages/api/src/telemetry/config.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
const DEFAULT_SERVICE_NAME = 'librechat';
|
||||
export const DEFAULT_HEALTH_PATH = '/health';
|
||||
|
||||
export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | 'stopped';
|
||||
|
||||
export interface TelemetryConfig {
|
||||
enabled: boolean;
|
||||
healthPath: string;
|
||||
sdkDisabled: boolean;
|
||||
serviceName: string;
|
||||
serviceVersion?: string;
|
||||
}
|
||||
|
||||
function isTruthy(value?: string | boolean | null): boolean {
|
||||
if (typeof value === 'boolean') {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
return value.trim().toLowerCase() === 'true';
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function normalizeEnvValue(value?: string): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): TelemetryConfig {
|
||||
const sdkDisabled = isTruthy(env.OTEL_SDK_DISABLED);
|
||||
const enabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled;
|
||||
const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;
|
||||
const serviceVersion =
|
||||
normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);
|
||||
|
||||
return {
|
||||
enabled,
|
||||
serviceName,
|
||||
sdkDisabled,
|
||||
serviceVersion,
|
||||
healthPath: DEFAULT_HEALTH_PATH,
|
||||
};
|
||||
}
|
||||
5
packages/api/src/telemetry/index.ts
Normal file
5
packages/api/src/telemetry/index.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export { getTelemetryConfig } from './config';
|
||||
export { initializeTelemetry, shutdownTelemetry } from './sdk';
|
||||
export { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
|
||||
export type { TelemetryConfig, TelemetryStatus } from './config';
|
||||
export type { TelemetryController } from './sdk';
|
||||
395
packages/api/src/telemetry/middleware.spec.ts
Normal file
395
packages/api/src/telemetry/middleware.spec.ts
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
import { EventEmitter } from 'node:events';
|
||||
import { SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import type { NextFunction, Response } from 'express';
|
||||
import type { Span } from '@opentelemetry/api';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { getTelemetryRequestSpan } from './sdk';
|
||||
import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
|
||||
|
||||
jest.mock('./sdk', () => ({
|
||||
getTelemetryRequestSpan: jest.fn(),
|
||||
}));
|
||||
|
||||
const mockGetTelemetryRequestSpan = getTelemetryRequestSpan as jest.MockedFunction<
|
||||
typeof getTelemetryRequestSpan
|
||||
>;
|
||||
|
||||
interface MockResponse extends EventEmitter {
|
||||
statusCode: number;
|
||||
writableEnded: boolean;
|
||||
}
|
||||
|
||||
function createSpan(): jest.Mocked<Span> {
|
||||
const span = {} as jest.Mocked<Span>;
|
||||
span.addEvent = jest.fn<jest.Mocked<Span>, Parameters<Span['addEvent']>>(() => span);
|
||||
span.addLink = jest.fn<jest.Mocked<Span>, Parameters<Span['addLink']>>(() => span);
|
||||
span.addLinks = jest.fn<jest.Mocked<Span>, Parameters<Span['addLinks']>>(() => span);
|
||||
span.end = jest.fn<void, Parameters<Span['end']>>();
|
||||
span.isRecording = jest.fn<boolean, Parameters<Span['isRecording']>>(() => true);
|
||||
span.recordException = jest.fn<void, Parameters<Span['recordException']>>();
|
||||
span.setAttribute = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttribute']>>(() => span);
|
||||
span.setAttributes = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttributes']>>(() => span);
|
||||
span.setStatus = jest.fn<jest.Mocked<Span>, Parameters<Span['setStatus']>>(() => span);
|
||||
span.spanContext = jest.fn<ReturnType<Span['spanContext']>, Parameters<Span['spanContext']>>(
|
||||
() => ({
|
||||
spanId: '0000000000000000',
|
||||
traceFlags: 0,
|
||||
traceId: '00000000000000000000000000000000',
|
||||
}),
|
||||
);
|
||||
span.updateName = jest.fn<jest.Mocked<Span>, Parameters<Span['updateName']>>(() => span);
|
||||
return span;
|
||||
}
|
||||
|
||||
function createResponse(statusCode = 200): MockResponse {
|
||||
const res = new EventEmitter() as MockResponse;
|
||||
res.statusCode = statusCode;
|
||||
res.writableEnded = false;
|
||||
return res;
|
||||
}
|
||||
|
||||
function createRequest(overrides: Partial<ServerRequest> = {}): ServerRequest {
|
||||
return {
|
||||
baseUrl: '/api/messages',
|
||||
body: {
|
||||
prompt: 'do not capture this prompt',
|
||||
text: 'do not capture this body',
|
||||
},
|
||||
headers: {
|
||||
authorization: 'Bearer do-not-capture-this-auth-header',
|
||||
cookie: 'session=do-not-capture-this-cookie',
|
||||
'x-api-key': 'do-not-capture-this-api-key',
|
||||
},
|
||||
method: 'POST',
|
||||
path: '/api/messages/conversation-1',
|
||||
route: { path: '/:conversationId' },
|
||||
user: {
|
||||
email: 'do-not-capture@example.com',
|
||||
id: 'user-1',
|
||||
tenantId: 'tenant-1',
|
||||
} as ServerRequest['user'],
|
||||
...overrides,
|
||||
} as ServerRequest;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
mockGetTelemetryRequestSpan.mockReset();
|
||||
});
|
||||
|
||||
describe('telemetryMiddleware', () => {
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('passes through without an active span', () => {
|
||||
const next = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);
|
||||
|
||||
telemetryMiddleware(createRequest(), createResponse() as Response, next);
|
||||
|
||||
expect(next).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('uses the stored request span for deferred completion attributes', () => {
|
||||
const activeSpan = createSpan();
|
||||
const requestSpan = createSpan();
|
||||
const req = createRequest();
|
||||
const res = createResponse(202);
|
||||
const next: NextFunction = jest.fn();
|
||||
mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);
|
||||
|
||||
telemetryMiddleware(req, res as Response, next);
|
||||
res.emit('finish');
|
||||
|
||||
expect(next).toHaveBeenCalledTimes(1);
|
||||
expect(trace.getActiveSpan).not.toHaveBeenCalled();
|
||||
expect(activeSpan.setAttributes).not.toHaveBeenCalled();
|
||||
expect(requestSpan.setAttributes).toHaveBeenCalledWith({
|
||||
'http.request.method': 'POST',
|
||||
});
|
||||
expect(requestSpan.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'http.response.status_code': 202,
|
||||
'http.route': '/api/messages/:conversationId',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('records safe route and identity attributes without body content', () => {
|
||||
const span = createSpan();
|
||||
const req = createRequest();
|
||||
const res = createResponse(201);
|
||||
const next: NextFunction = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(req, res as Response, next);
|
||||
res.emit('finish');
|
||||
|
||||
expect(next).toHaveBeenCalledTimes(1);
|
||||
expect(span.setAttributes).toHaveBeenCalledWith({
|
||||
'http.request.method': 'POST',
|
||||
});
|
||||
expect(span.setAttributes).toHaveBeenCalledWith({
|
||||
'enduser.id': 'user-1',
|
||||
'librechat.tenant.id': 'tenant-1',
|
||||
});
|
||||
expect(span.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'http.response.status_code': 201,
|
||||
'http.route': '/api/messages/:conversationId',
|
||||
}),
|
||||
);
|
||||
|
||||
const capturedAttributes = JSON.stringify(span.setAttributes.mock.calls);
|
||||
expect(capturedAttributes).not.toContain('do not capture this prompt');
|
||||
expect(capturedAttributes).not.toContain('do not capture this body');
|
||||
expect(capturedAttributes).not.toContain('do-not-capture@example.com');
|
||||
expect(capturedAttributes).not.toContain('conversation-1');
|
||||
expect(capturedAttributes).not.toContain('do-not-capture-this-auth-header');
|
||||
expect(capturedAttributes).not.toContain('do-not-capture-this-cookie');
|
||||
expect(capturedAttributes).not.toContain('do-not-capture-this-api-key');
|
||||
});
|
||||
|
||||
it('records identity attributes populated by downstream middleware', () => {
|
||||
const span = createSpan();
|
||||
const req = createRequest({
|
||||
headers: {},
|
||||
user: undefined,
|
||||
});
|
||||
const res = createResponse(200);
|
||||
const next: NextFunction = jest.fn(() => {
|
||||
req.user = {
|
||||
id: 'late-user',
|
||||
tenantId: 'late-tenant',
|
||||
} as ServerRequest['user'];
|
||||
});
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(req, res as Response, next);
|
||||
|
||||
expect(next).toHaveBeenCalledTimes(1);
|
||||
expect(span.setAttributes).toHaveBeenCalledTimes(1);
|
||||
|
||||
res.emit('finish');
|
||||
|
||||
expect(span.setAttributes).toHaveBeenCalledWith({
|
||||
'enduser.id': 'late-user',
|
||||
'librechat.tenant.id': 'late-tenant',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not derive tenant identity from request headers', () => {
|
||||
const span = createSpan();
|
||||
const req = createRequest({
|
||||
headers: {
|
||||
'x-tenant-id': 'spoofed-tenant',
|
||||
},
|
||||
user: undefined,
|
||||
});
|
||||
const res = createResponse(200);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(req, res as Response, jest.fn());
|
||||
res.emit('finish');
|
||||
|
||||
expect(span.setAttributes).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'librechat.tenant.id': 'spoofed-tenant',
|
||||
}),
|
||||
);
|
||||
expect(JSON.stringify(span.setAttributes.mock.calls)).not.toContain('spoofed-tenant');
|
||||
});
|
||||
|
||||
it('ignores health checks', () => {
|
||||
const span = createSpan();
|
||||
const next = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(
|
||||
createRequest({
|
||||
baseUrl: '',
|
||||
path: '/health',
|
||||
route: undefined,
|
||||
}),
|
||||
createResponse() as Response,
|
||||
next,
|
||||
);
|
||||
|
||||
expect(next).toHaveBeenCalledTimes(1);
|
||||
expect(span.setAttributes).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses a low-cardinality fallback for unmatched API routes', () => {
|
||||
const span = createSpan();
|
||||
const req = createRequest({
|
||||
baseUrl: '',
|
||||
path: '/api/nonexistent/123',
|
||||
route: undefined,
|
||||
});
|
||||
const res = createResponse(404);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(req, res as Response, jest.fn());
|
||||
res.emit('finish');
|
||||
|
||||
expect(span.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'http.route': '/api/*',
|
||||
'http.response.status_code': 404,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('uses a low-cardinality fallback for unmatched SPA routes', () => {
|
||||
const span = createSpan();
|
||||
const req = createRequest({
|
||||
baseUrl: '',
|
||||
path: '/chat/conversation-id',
|
||||
route: undefined,
|
||||
});
|
||||
const res = createResponse(200);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(req, res as Response, jest.fn());
|
||||
res.emit('finish');
|
||||
|
||||
expect(span.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'http.route': 'spa_fallback',
|
||||
'http.response.status_code': 200,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('marks server responses as errored', () => {
|
||||
const span = createSpan();
|
||||
const req = createRequest();
|
||||
const res = createResponse(500);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(req, res as Response, jest.fn());
|
||||
res.emit('finish');
|
||||
|
||||
expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
});
|
||||
|
||||
it('records completion attributes only once when finish and close both fire', () => {
|
||||
const span = createSpan();
|
||||
const res = createResponse(200);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(createRequest(), res as Response, jest.fn());
|
||||
res.emit('finish');
|
||||
res.emit('close');
|
||||
|
||||
expect(span.setAttributes).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('marks client disconnects before finish as aborted errors', () => {
|
||||
const span = createSpan();
|
||||
const res = createResponse(200);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryMiddleware(createRequest(), res as Response, jest.fn());
|
||||
res.emit('close');
|
||||
|
||||
expect(span.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'http.response.status_code': 499,
|
||||
'librechat.request.aborted': true,
|
||||
}),
|
||||
);
|
||||
expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
});
|
||||
});
|
||||
|
||||
describe('telemetryErrorMiddleware', () => {
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('records exceptions and forwards the error', () => {
|
||||
const span = createSpan();
|
||||
const error = new TypeError('boom');
|
||||
const next = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
|
||||
|
||||
expect(span.recordException).toHaveBeenCalledWith(error);
|
||||
expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
expect(span.setAttributes).toHaveBeenCalledWith({
|
||||
'enduser.id': 'user-1',
|
||||
'librechat.tenant.id': 'tenant-1',
|
||||
});
|
||||
expect(span.setAttributes).toHaveBeenCalledWith({
|
||||
'error.type': 'TypeError',
|
||||
'http.route': '/api/messages/:conversationId',
|
||||
});
|
||||
expect(next).toHaveBeenCalledWith(error);
|
||||
});
|
||||
|
||||
it('records exceptions on the stored request span when available', () => {
|
||||
const activeSpan = createSpan();
|
||||
const requestSpan = createSpan();
|
||||
const error = new TypeError('boom');
|
||||
const next = jest.fn();
|
||||
mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);
|
||||
|
||||
telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
|
||||
|
||||
expect(trace.getActiveSpan).not.toHaveBeenCalled();
|
||||
expect(activeSpan.recordException).not.toHaveBeenCalled();
|
||||
expect(requestSpan.recordException).toHaveBeenCalledWith(error);
|
||||
expect(requestSpan.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
expect(next).toHaveBeenCalledWith(error);
|
||||
});
|
||||
|
||||
it('handles non-Error values without throwing', () => {
|
||||
const span = createSpan();
|
||||
const next = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryErrorMiddleware('boom', createRequest(), createResponse() as Response, next);
|
||||
|
||||
expect(span.recordException).toHaveBeenCalledWith('boom');
|
||||
expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
expect(span.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'error.type': 'string',
|
||||
'http.route': '/api/messages/:conversationId',
|
||||
}),
|
||||
);
|
||||
expect(next).toHaveBeenCalledWith('boom');
|
||||
});
|
||||
|
||||
it('handles null error values without throwing', () => {
|
||||
const span = createSpan();
|
||||
const next = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
|
||||
|
||||
telemetryErrorMiddleware(null, createRequest(), createResponse() as Response, next);
|
||||
|
||||
expect(span.recordException).not.toHaveBeenCalled();
|
||||
expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
|
||||
expect(span.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
'error.type': 'null',
|
||||
'http.route': '/api/messages/:conversationId',
|
||||
}),
|
||||
);
|
||||
expect(next).toHaveBeenCalledWith(null);
|
||||
});
|
||||
|
||||
it('forwards the error without an active span', () => {
|
||||
const error = new Error('boom');
|
||||
const next = jest.fn();
|
||||
jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);
|
||||
|
||||
telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
|
||||
|
||||
expect(next).toHaveBeenCalledWith(error);
|
||||
});
|
||||
});
|
||||
171
packages/api/src/telemetry/middleware.ts
Normal file
171
packages/api/src/telemetry/middleware.ts
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
import { SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import type { Span, Attributes } from '@opentelemetry/api';
|
||||
import type { NextFunction, Response } from 'express';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { getTelemetryRequestSpan } from './sdk';
|
||||
import { DEFAULT_HEALTH_PATH } from './config';
|
||||
|
||||
const CLIENT_CLOSED_REQUEST_STATUS_CODE = 499;
|
||||
|
||||
type ExpressErrorValue =
|
||||
| Error
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| bigint
|
||||
| symbol
|
||||
| object
|
||||
| null
|
||||
| undefined;
|
||||
|
||||
function getUserId(req: ServerRequest): string | undefined {
|
||||
return req.user?.id;
|
||||
}
|
||||
|
||||
function getTenantId(req: ServerRequest): string | undefined {
|
||||
return req.user?.tenantId;
|
||||
}
|
||||
|
||||
function isHealthPath(req: ServerRequest): boolean {
|
||||
return req.path === DEFAULT_HEALTH_PATH;
|
||||
}
|
||||
|
||||
function isApiPath(req: ServerRequest): boolean {
|
||||
return req.path === '/api' || req.path.startsWith('/api/');
|
||||
}
|
||||
|
||||
function getRoutePath(req: ServerRequest): string {
|
||||
const routePath = req.route?.path;
|
||||
if (typeof routePath === 'string') {
|
||||
return `${req.baseUrl}${routePath}`;
|
||||
}
|
||||
|
||||
if (isHealthPath(req)) {
|
||||
return '/health';
|
||||
}
|
||||
|
||||
if (isApiPath(req)) {
|
||||
return '/api/*';
|
||||
}
|
||||
|
||||
return 'spa_fallback';
|
||||
}
|
||||
|
||||
function setIdentityAttributes(span: Span, req: ServerRequest): void {
|
||||
const userId = getUserId(req);
|
||||
const tenantId = getTenantId(req);
|
||||
|
||||
if (!userId && !tenantId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const attributes: Attributes = {};
|
||||
|
||||
if (userId) {
|
||||
attributes['enduser.id'] = userId;
|
||||
}
|
||||
|
||||
if (tenantId) {
|
||||
attributes['librechat.tenant.id'] = tenantId;
|
||||
}
|
||||
|
||||
span.setAttributes(attributes);
|
||||
}
|
||||
|
||||
function setCompletionAttributes(
|
||||
span: Span,
|
||||
req: ServerRequest,
|
||||
res: Response,
|
||||
aborted = false,
|
||||
): void {
|
||||
const statusCode = aborted ? CLIENT_CLOSED_REQUEST_STATUS_CODE : res.statusCode;
|
||||
const routePath = getRoutePath(req);
|
||||
const attributes: Attributes = {
|
||||
'http.route': routePath,
|
||||
'http.response.status_code': statusCode,
|
||||
};
|
||||
|
||||
if (aborted) {
|
||||
attributes['librechat.request.aborted'] = true;
|
||||
}
|
||||
|
||||
setIdentityAttributes(span, req);
|
||||
span.setAttributes(attributes);
|
||||
|
||||
if (aborted || statusCode >= 500) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR });
|
||||
}
|
||||
}
|
||||
|
||||
export function telemetryMiddleware(req: ServerRequest, res: Response, next: NextFunction): void {
|
||||
if (isHealthPath(req)) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();
|
||||
if (!span) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
span.setAttributes({
|
||||
'http.request.method': req.method,
|
||||
});
|
||||
|
||||
let completed = false;
|
||||
const complete = () => {
|
||||
if (completed) {
|
||||
return;
|
||||
}
|
||||
completed = true;
|
||||
setCompletionAttributes(span, req, res);
|
||||
};
|
||||
|
||||
const close = () => {
|
||||
if (completed) {
|
||||
return;
|
||||
}
|
||||
completed = true;
|
||||
setCompletionAttributes(span, req, res, !res.writableEnded);
|
||||
};
|
||||
|
||||
res.once('finish', complete);
|
||||
res.once('close', close);
|
||||
next();
|
||||
}
|
||||
|
||||
export function telemetryErrorMiddleware(
|
||||
err: ExpressErrorValue,
|
||||
req: ServerRequest,
|
||||
_res: Response,
|
||||
next: NextFunction,
|
||||
): void {
|
||||
const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();
|
||||
if (span) {
|
||||
const routePath = getRoutePath(req);
|
||||
if (err) {
|
||||
span.recordException(err instanceof Error ? err : String(err));
|
||||
}
|
||||
span.setStatus({ code: SpanStatusCode.ERROR });
|
||||
setIdentityAttributes(span, req);
|
||||
span.setAttributes({
|
||||
'error.type': getErrorType(err),
|
||||
'http.route': routePath,
|
||||
});
|
||||
}
|
||||
|
||||
next(err);
|
||||
}
|
||||
|
||||
function getErrorType(err: ExpressErrorValue): string {
|
||||
if (err instanceof Error) {
|
||||
return err.name || err.constructor.name;
|
||||
}
|
||||
|
||||
if (err === null) {
|
||||
return 'null';
|
||||
}
|
||||
|
||||
return typeof err;
|
||||
}
|
||||
357
packages/api/src/telemetry/sdk.spec.ts
Normal file
357
packages/api/src/telemetry/sdk.spec.ts
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
import { Socket } from 'node:net';
|
||||
import { IncomingMessage } from 'node:http';
|
||||
import type { Span } from '@opentelemetry/api';
|
||||
|
||||
interface HttpInstrumentationOptions {
|
||||
requestHook?: (span: Span, request: object) => void;
|
||||
startIncomingSpanHook?: (request: IncomingMessage) => Record<string, string>;
|
||||
}
|
||||
|
||||
const mockStart = jest.fn();
|
||||
const mockShutdown = jest.fn();
|
||||
const mockNodeSDK = jest.fn(() => ({
|
||||
start: mockStart,
|
||||
shutdown: mockShutdown,
|
||||
}));
|
||||
const mockExpressInstrumentation = jest.fn(() => ({ name: 'express' }));
|
||||
const mockHttpInstrumentation = jest.fn((options?: HttpInstrumentationOptions) => ({
|
||||
name: 'http',
|
||||
options,
|
||||
}));
|
||||
const mockIORedisInstrumentation = jest.fn(() => ({ name: 'ioredis' }));
|
||||
const mockMongoDBInstrumentation = jest.fn(() => ({ name: 'mongodb' }));
|
||||
const mockMongooseInstrumentation = jest.fn(() => ({ name: 'mongoose' }));
|
||||
const mockUndiciInstrumentation = jest.fn(() => ({ name: 'undici' }));
|
||||
const mockResourceFromAttributes = jest.fn((attributes: object) => ({ attributes }));
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/sdk-node',
|
||||
() => ({
|
||||
NodeSDK: mockNodeSDK,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/instrumentation-express',
|
||||
() => ({
|
||||
ExpressInstrumentation: mockExpressInstrumentation,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/instrumentation-http',
|
||||
() => ({
|
||||
HttpInstrumentation: mockHttpInstrumentation,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/instrumentation-ioredis',
|
||||
() => ({
|
||||
IORedisInstrumentation: mockIORedisInstrumentation,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/instrumentation-mongodb',
|
||||
() => ({
|
||||
MongoDBInstrumentation: mockMongoDBInstrumentation,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/instrumentation-mongoose',
|
||||
() => ({
|
||||
MongooseInstrumentation: mockMongooseInstrumentation,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/instrumentation-undici',
|
||||
() => ({
|
||||
UndiciInstrumentation: mockUndiciInstrumentation,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/resources',
|
||||
() => ({
|
||||
resourceFromAttributes: mockResourceFromAttributes,
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'@opentelemetry/semantic-conventions',
|
||||
() => ({
|
||||
ATTR_SERVICE_NAME: 'service.name',
|
||||
ATTR_SERVICE_VERSION: 'service.version',
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
async function flushSignalShutdown(): Promise<void> {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
describe('telemetry SDK lifecycle', () => {
|
||||
let emitWarningSpy: jest.SpyInstance;
|
||||
let getTelemetryRequestSpan: (typeof import('./sdk'))['getTelemetryRequestSpan'];
|
||||
let initializeTelemetry: (typeof import('./sdk'))['initializeTelemetry'];
|
||||
let resetTelemetryForTests: (typeof import('./sdk'))['resetTelemetryForTests'];
|
||||
let shutdownTelemetry: (typeof import('./sdk'))['shutdownTelemetry'];
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks();
|
||||
({ getTelemetryRequestSpan, initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } =
|
||||
await import('./sdk'));
|
||||
await resetTelemetryForTests();
|
||||
Reflect.deleteProperty(globalThis, 'Bun');
|
||||
emitWarningSpy = jest.spyOn(process, 'emitWarning').mockImplementation(() => true);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await resetTelemetryForTests();
|
||||
emitWarningSpy.mockRestore();
|
||||
Reflect.deleteProperty(globalThis, 'Bun');
|
||||
});
|
||||
|
||||
it('does not initialize when tracing is disabled by default', () => {
|
||||
const controller = initializeTelemetry({});
|
||||
|
||||
expect(controller.enabled).toBe(false);
|
||||
expect(controller.status).toBe('disabled');
|
||||
expect(mockNodeSDK).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not initialize when OTEL_SDK_DISABLED is true', () => {
|
||||
const controller = initializeTelemetry({
|
||||
OTEL_SDK_DISABLED: 'true',
|
||||
OTEL_TRACING_ENABLED: 'true',
|
||||
});
|
||||
|
||||
expect(controller.enabled).toBe(false);
|
||||
expect(controller.status).toBe('disabled');
|
||||
expect(mockNodeSDK).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not initialize under Bun runtime', () => {
|
||||
Object.defineProperty(globalThis, 'Bun', {
|
||||
configurable: true,
|
||||
value: {},
|
||||
});
|
||||
|
||||
const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
expect(controller.enabled).toBe(false);
|
||||
expect(controller.status).toBe('disabled');
|
||||
expect(mockNodeSDK).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('starts the Node SDK once when enabled', () => {
|
||||
const first = initializeTelemetry({
|
||||
OTEL_SERVICE_NAME: 'librechat-test',
|
||||
OTEL_TRACING_ENABLED: 'true',
|
||||
});
|
||||
const second = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
expect(first.enabled).toBe(true);
|
||||
expect(first.status).toBe('started');
|
||||
expect(second.enabled).toBe(true);
|
||||
expect(mockNodeSDK).toHaveBeenCalledTimes(1);
|
||||
expect(mockStart).toHaveBeenCalledTimes(1);
|
||||
expect(mockResourceFromAttributes).toHaveBeenCalledWith({
|
||||
'service.name': 'librechat-test',
|
||||
});
|
||||
expect(mockHttpInstrumentation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
headersToSpanAttributes: {
|
||||
client: { requestHeaders: [], responseHeaders: [] },
|
||||
server: { requestHeaders: [], responseHeaders: [] },
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1);
|
||||
expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1);
|
||||
expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1);
|
||||
expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1);
|
||||
expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('tracks HTTP server request spans for completion updates', () => {
|
||||
const span = {} as Span;
|
||||
const request = new IncomingMessage(new Socket());
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const requestHook = instrumentationOptions?.requestHook;
|
||||
|
||||
if (!requestHook) {
|
||||
throw new Error('HTTP instrumentation requestHook was not configured');
|
||||
}
|
||||
|
||||
requestHook(span, request);
|
||||
|
||||
expect(getTelemetryRequestSpan(request)).toBe(span);
|
||||
});
|
||||
|
||||
it('redacts incoming URL attributes before HTTP spans are exported', () => {
|
||||
const request = new IncomingMessage(new Socket());
|
||||
request.url = '/oauth/callback?code=secret-code&state=secret-state';
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startIncomingSpanHook = instrumentationOptions?.startIncomingSpanHook;
|
||||
|
||||
if (!startIncomingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startIncomingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startIncomingSpanHook(request);
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': 'spa_fallback?[REDACTED]',
|
||||
'http.url': 'spa_fallback?[REDACTED]',
|
||||
'url.full': 'spa_fallback?[REDACTED]',
|
||||
'url.path': 'spa_fallback',
|
||||
'url.query': '[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('secret-code');
|
||||
expect(JSON.stringify(attributes)).not.toContain('secret-state');
|
||||
});
|
||||
|
||||
it('reflects lifecycle status from the controller getter', async () => {
|
||||
const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
expect(controller.status).toBe('started');
|
||||
await controller.shutdown();
|
||||
expect(controller.status).toBe('stopped');
|
||||
expect(controller.enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('handles async SDK start failures without throwing', async () => {
|
||||
mockStart.mockRejectedValueOnce(new Error('async start failed'));
|
||||
|
||||
const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
expect(controller.enabled).toBe(true);
|
||||
expect(controller.status).toBe('starting');
|
||||
await controller.shutdown();
|
||||
expect(controller.enabled).toBe(false);
|
||||
expect(controller.status).toBe('failed');
|
||||
expect(emitWarningSpy).toHaveBeenCalledWith(
|
||||
'OpenTelemetry initialization failed: async start failed',
|
||||
{ code: 'LIBRECHAT_OTEL' },
|
||||
);
|
||||
});
|
||||
|
||||
it('returns failed status without throwing when SDK start fails', () => {
|
||||
mockStart.mockImplementationOnce(() => {
|
||||
throw new Error('start failed');
|
||||
});
|
||||
|
||||
const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
expect(controller.enabled).toBe(false);
|
||||
expect(controller.status).toBe('failed');
|
||||
expect(emitWarningSpy).toHaveBeenCalledWith(
|
||||
'OpenTelemetry initialization failed: start failed',
|
||||
{ code: 'LIBRECHAT_OTEL' },
|
||||
);
|
||||
});
|
||||
|
||||
it('shuts down the active SDK idempotently', async () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
await shutdownTelemetry();
|
||||
await shutdownTelemetry();
|
||||
|
||||
expect(mockShutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('coalesces concurrent shutdown calls', async () => {
|
||||
let resolveShutdown: () => void = () => undefined;
|
||||
mockShutdown.mockReturnValueOnce(
|
||||
new Promise<void>((resolve) => {
|
||||
resolveShutdown = resolve;
|
||||
}),
|
||||
);
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
const firstShutdown = shutdownTelemetry();
|
||||
const secondShutdown = shutdownTelemetry();
|
||||
expect(mockShutdown).toHaveBeenCalledTimes(1);
|
||||
|
||||
resolveShutdown();
|
||||
await Promise.all([firstShutdown, secondShutdown]);
|
||||
});
|
||||
|
||||
it('keeps the active SDK available when shutdown fails', async () => {
|
||||
mockShutdown.mockRejectedValueOnce(new Error('shutdown failed'));
|
||||
const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
await expect(shutdownTelemetry()).rejects.toThrow('shutdown failed');
|
||||
expect(controller.status).toBe('started');
|
||||
|
||||
await shutdownTelemetry();
|
||||
expect(mockShutdown).toHaveBeenCalledTimes(2);
|
||||
expect(controller.status).toBe('stopped');
|
||||
});
|
||||
|
||||
it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(
|
||||
'does not force process exit when another %s handler is registered',
|
||||
async (signal) => {
|
||||
const otherHandler = jest.fn();
|
||||
const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
process.once(signal, otherHandler);
|
||||
|
||||
process.emit(signal, signal);
|
||||
await flushSignalShutdown();
|
||||
|
||||
expect(mockShutdown).toHaveBeenCalledTimes(1);
|
||||
expect(otherHandler).toHaveBeenCalledTimes(1);
|
||||
expect(killSpy).not.toHaveBeenCalled();
|
||||
|
||||
killSpy.mockRestore();
|
||||
},
|
||||
);
|
||||
|
||||
it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(
|
||||
'reraises the shutdown %s signal when telemetry is the only signal handler',
|
||||
async (signal) => {
|
||||
const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
process.emit(signal, signal);
|
||||
await flushSignalShutdown();
|
||||
|
||||
expect(mockShutdown).toHaveBeenCalledTimes(1);
|
||||
expect(killSpy).toHaveBeenCalledWith(process.pid, signal);
|
||||
|
||||
killSpy.mockRestore();
|
||||
},
|
||||
);
|
||||
|
||||
it('warns and reraises the signal when shutdown rejects', async () => {
|
||||
mockShutdown.mockRejectedValueOnce(new Error('signal shutdown failed'));
|
||||
const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
process.emit('SIGTERM', 'SIGTERM');
|
||||
await flushSignalShutdown();
|
||||
|
||||
expect(mockShutdown).toHaveBeenCalledTimes(1);
|
||||
expect(emitWarningSpy).toHaveBeenCalledWith(
|
||||
'OpenTelemetry shutdown failed: signal shutdown failed',
|
||||
{ code: 'LIBRECHAT_OTEL' },
|
||||
);
|
||||
expect(killSpy).toHaveBeenCalledWith(process.pid, 'SIGTERM');
|
||||
|
||||
killSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
328
packages/api/src/telemetry/sdk.ts
Normal file
328
packages/api/src/telemetry/sdk.ts
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
import { IncomingMessage } from 'node:http';
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
|
||||
import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici';
|
||||
import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
|
||||
import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';
|
||||
import { MongoDBInstrumentation } from '@opentelemetry/instrumentation-mongodb';
|
||||
import { MongooseInstrumentation } from '@opentelemetry/instrumentation-mongoose';
|
||||
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
||||
import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node';
|
||||
import type { Span, Attributes } from '@opentelemetry/api';
|
||||
import type { TelemetryConfig, TelemetryStatus } from './config';
|
||||
import { getTelemetryConfig } from './config';
|
||||
|
||||
export interface TelemetryController {
|
||||
readonly enabled: boolean;
|
||||
readonly status: TelemetryStatus;
|
||||
shutdown: () => Promise<void>;
|
||||
}
|
||||
|
||||
const WARNING_CODE = 'LIBRECHAT_OTEL';
|
||||
const REDACTED_QUERY_VALUE = '[REDACTED]';
|
||||
const SIGNAL_SHUTDOWN_TIMEOUT_MS = 5_000;
|
||||
|
||||
interface RegisteredSignal {
|
||||
signal: NodeJS.Signals;
|
||||
listener: NodeJS.SignalsListener;
|
||||
}
|
||||
|
||||
let activeSdk: NodeSDK | undefined;
|
||||
let pendingSdk: NodeSDK | undefined;
|
||||
let startPromise: Promise<void> | undefined;
|
||||
let shutdownPromise: Promise<void> | undefined;
|
||||
let status: TelemetryStatus = 'stopped';
|
||||
let registeredSignals: RegisteredSignal[] = [];
|
||||
let requestSpans = new WeakMap<IncomingMessage, Span>();
|
||||
|
||||
function isBunRuntime(): boolean {
|
||||
return Reflect.get(globalThis, 'Bun') != null;
|
||||
}
|
||||
|
||||
function shouldIgnoreIncomingRequest(request: IncomingMessage, healthPath: string): boolean {
|
||||
return request.url === healthPath || request.url?.startsWith(`${healthPath}?`) === true;
|
||||
}
|
||||
|
||||
function getIncomingUrlInfo(request: IncomingMessage): { hasQuery: boolean; pathname: string } {
|
||||
const rawUrl = request.url ?? '/';
|
||||
|
||||
try {
|
||||
const parsedUrl = new URL(rawUrl, 'http://localhost');
|
||||
return {
|
||||
hasQuery: parsedUrl.search.length > 1,
|
||||
pathname: parsedUrl.pathname || '/',
|
||||
};
|
||||
} catch {
|
||||
const queryIndex = rawUrl.indexOf('?');
|
||||
return {
|
||||
hasQuery: queryIndex >= 0 && queryIndex < rawUrl.length - 1,
|
||||
pathname: queryIndex >= 0 ? rawUrl.slice(0, queryIndex) || '/' : rawUrl || '/',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function getLowCardinalityUrlPath(pathname: string, healthPath: string): string {
|
||||
if (pathname === healthPath) {
|
||||
return healthPath;
|
||||
}
|
||||
|
||||
if (pathname === '/api' || pathname.startsWith('/api/')) {
|
||||
return '/api/*';
|
||||
}
|
||||
|
||||
return 'spa_fallback';
|
||||
}
|
||||
|
||||
function getSanitizedIncomingUrlAttributes(
|
||||
request: IncomingMessage,
|
||||
healthPath: string,
|
||||
): Attributes {
|
||||
const { hasQuery, pathname } = getIncomingUrlInfo(request);
|
||||
const safePath = getLowCardinalityUrlPath(pathname, healthPath);
|
||||
const safeTarget = hasQuery ? `${safePath}?${REDACTED_QUERY_VALUE}` : safePath;
|
||||
const attributes: Attributes = {
|
||||
'http.target': safeTarget,
|
||||
'http.url': safeTarget,
|
||||
'url.full': safeTarget,
|
||||
'url.path': safePath,
|
||||
};
|
||||
|
||||
if (hasQuery) {
|
||||
attributes['url.query'] = REDACTED_QUERY_VALUE;
|
||||
}
|
||||
|
||||
return attributes;
|
||||
}
|
||||
|
||||
function getResourceAttributes(config: TelemetryConfig): Attributes {
|
||||
const attributes: Attributes = {
|
||||
[ATTR_SERVICE_NAME]: config.serviceName,
|
||||
};
|
||||
|
||||
if (config.serviceVersion) {
|
||||
attributes[ATTR_SERVICE_VERSION] = config.serviceVersion;
|
||||
}
|
||||
|
||||
return attributes;
|
||||
}
|
||||
|
||||
function createSdk(config: TelemetryConfig): NodeSDK {
|
||||
const sdkConfig: Partial<NodeSDKConfiguration> = {
|
||||
resource: resourceFromAttributes(getResourceAttributes(config)),
|
||||
instrumentations: [
|
||||
new HttpInstrumentation({
|
||||
headersToSpanAttributes: {
|
||||
client: { requestHeaders: [], responseHeaders: [] },
|
||||
server: { requestHeaders: [], responseHeaders: [] },
|
||||
},
|
||||
requestHook: (span: Span, request: object) => {
|
||||
if (request instanceof IncomingMessage) {
|
||||
requestSpans.set(request, span);
|
||||
}
|
||||
},
|
||||
startIncomingSpanHook: (request: IncomingMessage) =>
|
||||
getSanitizedIncomingUrlAttributes(request, config.healthPath),
|
||||
ignoreIncomingRequestHook: (request: IncomingMessage) =>
|
||||
shouldIgnoreIncomingRequest(request, config.healthPath),
|
||||
}),
|
||||
new ExpressInstrumentation(),
|
||||
new MongoDBInstrumentation(),
|
||||
new MongooseInstrumentation(),
|
||||
new IORedisInstrumentation(),
|
||||
new UndiciInstrumentation(),
|
||||
],
|
||||
};
|
||||
|
||||
return new NodeSDK(sdkConfig);
|
||||
}
|
||||
|
||||
export function getTelemetryRequestSpan(request: IncomingMessage): Span | undefined {
|
||||
return requestSpans.get(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* NodeSDK.start has been synchronous in some supported OpenTelemetry versions
|
||||
* and promise-returning in others, so the lifecycle wrapper accepts either form.
|
||||
*/
|
||||
function startSdk(sdk: NodeSDK): void | Promise<void> {
|
||||
return (sdk as NodeSDK & { start: () => void | Promise<void> }).start();
|
||||
}
|
||||
|
||||
function emitWarning(message: string): void {
|
||||
process.emitWarning(message, { code: WARNING_CODE });
|
||||
}
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isControllerEnabled(): boolean {
|
||||
return status === 'starting' || status === 'started';
|
||||
}
|
||||
|
||||
function makeController(): TelemetryController {
|
||||
return {
|
||||
get enabled() {
|
||||
return isControllerEnabled();
|
||||
},
|
||||
get status() {
|
||||
return status;
|
||||
},
|
||||
shutdown: shutdownTelemetry,
|
||||
};
|
||||
}
|
||||
|
||||
function unregisterShutdownHandlers(): void {
|
||||
for (const { signal, listener } of registeredSignals) {
|
||||
process.removeListener(signal, listener);
|
||||
}
|
||||
registeredSignals = [];
|
||||
}
|
||||
|
||||
function registerShutdownHandlers(): void {
|
||||
if (registeredSignals.length > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT'];
|
||||
registeredSignals = signals.map((signal) => {
|
||||
const listener: NodeJS.SignalsListener = () => {
|
||||
const shouldReraiseSignal = process.listenerCount(signal) === 0;
|
||||
withTimeout(shutdownTelemetry(), SIGNAL_SHUTDOWN_TIMEOUT_MS)
|
||||
.catch((error) => {
|
||||
emitWarning(`OpenTelemetry shutdown failed: ${getErrorMessage(error)}`);
|
||||
})
|
||||
.finally(() => {
|
||||
if (shouldReraiseSignal) {
|
||||
process.kill(process.pid, signal);
|
||||
}
|
||||
});
|
||||
};
|
||||
process.once(signal, listener);
|
||||
return { signal, listener };
|
||||
});
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
reject(new Error(`timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
timeout.unref?.();
|
||||
});
|
||||
|
||||
return Promise.race([promise, timeoutPromise]).finally(() => {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function initializeTelemetry(env: NodeJS.ProcessEnv = process.env): TelemetryController {
|
||||
if (activeSdk || pendingSdk) {
|
||||
return makeController();
|
||||
}
|
||||
|
||||
const config = getTelemetryConfig(env);
|
||||
if (!config.enabled || isBunRuntime()) {
|
||||
status = 'disabled';
|
||||
return makeController();
|
||||
}
|
||||
|
||||
try {
|
||||
const sdk = createSdk(config);
|
||||
const result = startSdk(sdk);
|
||||
if (result) {
|
||||
pendingSdk = sdk;
|
||||
status = 'starting';
|
||||
const pendingStart = result
|
||||
.then(() => {
|
||||
if (pendingSdk === sdk) {
|
||||
pendingSdk = undefined;
|
||||
activeSdk = sdk;
|
||||
status = 'started';
|
||||
registerShutdownHandlers();
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
if (pendingSdk === sdk) {
|
||||
pendingSdk = undefined;
|
||||
status = 'failed';
|
||||
emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);
|
||||
}
|
||||
});
|
||||
startPromise = pendingStart;
|
||||
void pendingStart.finally(() => {
|
||||
if (startPromise === pendingStart) {
|
||||
startPromise = undefined;
|
||||
}
|
||||
});
|
||||
return makeController();
|
||||
}
|
||||
|
||||
activeSdk = sdk;
|
||||
status = 'started';
|
||||
registerShutdownHandlers();
|
||||
return makeController();
|
||||
} catch (error) {
|
||||
status = 'failed';
|
||||
emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);
|
||||
return makeController();
|
||||
}
|
||||
}
|
||||
|
||||
async function performShutdownTelemetry(): Promise<void> {
|
||||
if (startPromise) {
|
||||
await startPromise;
|
||||
}
|
||||
|
||||
if (!activeSdk) {
|
||||
status = status === 'started' ? 'stopped' : status;
|
||||
return;
|
||||
}
|
||||
|
||||
const sdk = activeSdk;
|
||||
try {
|
||||
await sdk.shutdown();
|
||||
activeSdk = undefined;
|
||||
status = 'stopped';
|
||||
unregisterShutdownHandlers();
|
||||
} catch (error) {
|
||||
status = 'started';
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function shutdownTelemetry(): Promise<void> {
|
||||
if (!shutdownPromise) {
|
||||
shutdownPromise = performShutdownTelemetry().finally(() => {
|
||||
shutdownPromise = undefined;
|
||||
});
|
||||
}
|
||||
|
||||
return shutdownPromise;
|
||||
}
|
||||
|
||||
export async function resetTelemetryForTests(): Promise<void> {
|
||||
try {
|
||||
if (startPromise) {
|
||||
await startPromise.catch(() => undefined);
|
||||
}
|
||||
|
||||
if (shutdownPromise) {
|
||||
await shutdownPromise.catch(() => undefined);
|
||||
} else if (activeSdk) {
|
||||
await Promise.resolve(activeSdk.shutdown()).catch(() => undefined);
|
||||
}
|
||||
} finally {
|
||||
activeSdk = undefined;
|
||||
pendingSdk = undefined;
|
||||
startPromise = undefined;
|
||||
shutdownPromise = undefined;
|
||||
status = 'stopped';
|
||||
requestSpans = new WeakMap<IncomingMessage, Span>();
|
||||
unregisterShutdownHandlers();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue