diff --git a/.env.example b/.env.example index 816b4cd14d..e1608ec907 100644 --- a/.env.example +++ b/.env.example @@ -110,6 +110,23 @@ NODE_MAX_OLD_SPACE_SIZE=6144 # LANGFUSE_SECRET_KEY= # LANGFUSE_BASE_URL= +#=======================# +# OpenTelemetry Tracing # +#=======================# + +# Enables backend OpenTelemetry tracing. General backend visibility only; +# use Langfuse for GenAI-specific prompt/model observability. +# OTEL_TRACING_ENABLED=false +# OTEL_SERVICE_NAME=librechat +# OTEL_SERVICE_VERSION= +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= +# OTEL_EXPORTER_OTLP_HEADERS= +# OTEL_TRACES_EXPORTER=otlp +# OTEL_TRACES_SAMPLER=parentbased_always_on +# OTEL_LOG_LEVEL=INFO +# OTEL_SDK_DISABLED=false + #===================================================# # Endpoints # #===================================================# diff --git a/api/package.json b/api/package.json index 80c3e5fd9b..8b3ff32a30 100644 --- a/api/package.json +++ b/api/package.json @@ -52,6 +52,16 @@ "@microsoft/microsoft-graph-client": "^3.0.7", "@modelcontextprotocol/sdk": "^1.29.0", "@node-saml/passport-saml": "^5.1.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation-express": "^0.56.0", + "@opentelemetry/instrumentation-http": "^0.207.0", + "@opentelemetry/instrumentation-ioredis": "^0.55.0", + "@opentelemetry/instrumentation-mongodb": "^0.60.0", + "@opentelemetry/instrumentation-mongoose": "^0.54.0", + "@opentelemetry/instrumentation-undici": "^0.18.0", + "@opentelemetry/resources": "^2.6.1", + "@opentelemetry/sdk-node": "^0.207.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@smithy/node-http-handler": "^4.4.5", "ai-tokenizer": "^1.0.6", "axios": "^1.16.0", diff --git a/api/server/index.js b/api/server/index.js index a8146fbc1c..9e094724f7 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -1,4 +1,4 @@ -require('dotenv').config(); +const telemetry = require('./telemetry'); const fs = require('fs'); const path = require('path'); require('module-alias')({ base: path.resolve(__dirname, '..') }); @@ -25,22 +25,22 @@ const { updateInterfacePermissions, } = require('@librechat/api'); const { connectDb, indexSync } = require('~/db'); -const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); const { - getRoleByName, updateAccessPermissions, - seedDatabase, sweepOrphanedPreviews, + getRoleByName, + seedDatabase, } = require('~/models'); +const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); const { capabilityContextMiddleware } = require('./middleware/roles/capabilities'); const createValidateImageRequest = require('./middleware/validateImageRequest'); const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies'); const { checkMigrations } = require('./services/start/migration'); +const optionalJwtAuth = require('./middleware/optionalJwtAuth'); const initializeMCPs = require('./services/initializeMCPs'); const configureSocialLogins = require('./socialLogins'); const { getAppConfig } = require('./services/Config'); const staticCache = require('./utils/staticCache'); -const optionalJwtAuth = require('./middleware/optionalJwtAuth'); const noIndex = require('./middleware/noIndex'); const routes = require('./routes'); @@ -146,6 +146,10 @@ const startServer = async () => { app.use(staticCache(appConfig.paths.fonts)); app.use(staticCache(appConfig.paths.assets)); + if (telemetry.enabled) { + app.use(telemetry.telemetryMiddleware); + } + if (!ALLOW_SOCIAL_LOGIN) { console.warn('Social logins are disabled. Set ALLOW_SOCIAL_LOGIN=true to enable them.'); } @@ -227,6 +231,10 @@ const startServer = async () => { res.send(updatedIndexHtml); }); + /** Record trace errors before the final error controller. */ + if (telemetry.enabled) { + app.use(telemetry.telemetryErrorMiddleware); + } /** Error handler (must be last - Express identifies error middleware by its 4-arg signature) */ app.use(ErrorController); diff --git a/api/server/index.spec.js b/api/server/index.spec.js index 7b3d062fce..573770e282 100644 --- a/api/server/index.spec.js +++ b/api/server/index.spec.js @@ -1,4 +1,5 @@ const fs = require('fs'); +const path = require('path'); const request = require('supertest'); const { MongoMemoryServer } = require('mongodb-memory-server'); const mongoose = require('mongoose'); @@ -32,6 +33,56 @@ jest.mock('~/config', () => ({ }), })); +jest.mock( + '@librechat/api/telemetry', + () => ({ + initializeTelemetry: jest.fn(() => ({ + enabled: false, + status: 'disabled', + shutdown: jest.fn(), + })), + telemetryMiddleware: jest.fn((_req, _res, next) => next()), + telemetryErrorMiddleware: jest.fn((err, _req, _res, next) => next(err)), + }), + { virtual: true }, +); + +describe('Telemetry wiring', () => { + const source = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8'); + + it('loads telemetry before other server imports', () => { + const firstStatement = source + .split('\n') + .map((line) => line.trim()) + .find(Boolean); + + expect(firstStatement).toBe("const telemetry = require('./telemetry');"); + }); + + it('mounts telemetry middleware after static assets and before routes', () => { + const telemetryMiddlewareIndex = source.indexOf('app.use(telemetry.telemetryMiddleware);'); + const staticAssetsIndex = source.indexOf('app.use(staticCache(appConfig.paths.assets));'); + const apiRoutesIndex = source.indexOf("app.use('/api/auth'"); + + expect(telemetryMiddlewareIndex).toBeGreaterThan(-1); + expect(staticAssetsIndex).toBeGreaterThan(-1); + expect(apiRoutesIndex).toBeGreaterThan(-1); + expect(staticAssetsIndex).toBeLessThan(telemetryMiddlewareIndex); + expect(telemetryMiddlewareIndex).toBeLessThan(apiRoutesIndex); + }); + + it('mounts telemetry error middleware before ErrorController', () => { + const telemetryErrorMiddlewareIndex = source.indexOf( + 'app.use(telemetry.telemetryErrorMiddleware);', + ); + const errorControllerIndex = source.indexOf('app.use(ErrorController);'); + + expect(telemetryErrorMiddlewareIndex).toBeGreaterThan(-1); + expect(errorControllerIndex).toBeGreaterThan(-1); + expect(telemetryErrorMiddlewareIndex).toBeLessThan(errorControllerIndex); + }); +}); + describe('Server Configuration', () => { // Increase the default timeout to allow for Mongo cleanup jest.setTimeout(30_000); diff --git a/api/server/telemetry.js b/api/server/telemetry.js new file mode 100644 index 0000000000..cb0058355b --- /dev/null +++ b/api/server/telemetry.js @@ -0,0 +1,40 @@ +require('dotenv').config(); + +function isTruthy(value) { + return value?.trim().toLowerCase() === 'true'; +} + +function isTelemetryEnabled() { + return isTruthy(process.env.OTEL_TRACING_ENABLED) && !isTruthy(process.env.OTEL_SDK_DISABLED); +} + +if (isTelemetryEnabled()) { + const { + initializeTelemetry, + telemetryMiddleware, + telemetryErrorMiddleware, + } = require('@librechat/api/telemetry'); + const controller = initializeTelemetry(); + + module.exports = { + get enabled() { + return controller.enabled; + }, + get status() { + return controller.status; + }, + shutdown: controller.shutdown, + telemetryMiddleware, + telemetryErrorMiddleware, + }; +} else { + module.exports = { + enabled: false, + get status() { + return 'disabled'; + }, + shutdown: async () => {}, + telemetryMiddleware: (_req, _res, next) => next(), + telemetryErrorMiddleware: (err, _req, _res, next) => next(err), + }; +} diff --git a/api/server/telemetry.spec.js b/api/server/telemetry.spec.js new file mode 100644 index 0000000000..8f848da8e7 --- /dev/null +++ b/api/server/telemetry.spec.js @@ -0,0 +1,91 @@ +describe('telemetry bootstrap', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { ...originalEnv }; + delete process.env.OTEL_SDK_DISABLED; + delete process.env.OTEL_TRACING_ENABLED; + jest.doMock('dotenv', () => ({ + config: jest.fn(), + })); + }); + + afterEach(() => { + process.env = originalEnv; + jest.dontMock('dotenv'); + jest.dontMock('@librechat/api/telemetry'); + jest.resetModules(); + }); + + it('does not load OpenTelemetry packages by default', () => { + jest.doMock( + '@librechat/api/telemetry', + () => { + throw new Error('telemetry package should not load when tracing is disabled'); + }, + { virtual: true }, + ); + + const telemetry = require('./telemetry'); + + expect(telemetry.enabled).toBe(false); + expect(telemetry.status).toBe('disabled'); + }); + + it('does not load OpenTelemetry packages when the SDK is disabled', () => { + process.env.OTEL_SDK_DISABLED = 'true'; + process.env.OTEL_TRACING_ENABLED = 'true'; + jest.doMock( + '@librechat/api/telemetry', + () => { + throw new Error('telemetry package should not load when the SDK is disabled'); + }, + { virtual: true }, + ); + + const telemetry = require('./telemetry'); + + expect(telemetry.enabled).toBe(false); + expect(telemetry.status).toBe('disabled'); + }); + + it('loads and exposes telemetry middleware when tracing is enabled', () => { + process.env.OTEL_TRACING_ENABLED = 'true'; + let enabled = true; + let status = 'starting'; + const telemetryMiddleware = jest.fn(); + const telemetryErrorMiddleware = jest.fn(); + const initializeTelemetry = jest.fn(() => ({ + get enabled() { + return enabled; + }, + get status() { + return status; + }, + shutdown: jest.fn(), + })); + jest.doMock( + '@librechat/api/telemetry', + () => ({ + initializeTelemetry, + telemetryMiddleware, + telemetryErrorMiddleware, + }), + { virtual: true }, + ); + + const telemetry = require('./telemetry'); + + expect(initializeTelemetry).toHaveBeenCalledTimes(1); + expect(telemetry.enabled).toBe(true); + expect(telemetry.status).toBe('starting'); + expect(telemetry.telemetryMiddleware).toBe(telemetryMiddleware); + expect(telemetry.telemetryErrorMiddleware).toBe(telemetryErrorMiddleware); + + enabled = false; + status = 'failed'; + expect(telemetry.enabled).toBe(false); + expect(telemetry.status).toBe('failed'); + }); +}); diff --git a/package-lock.json b/package-lock.json index dbc9fb994c..c25619187e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,6 +67,16 @@ "@microsoft/microsoft-graph-client": "^3.0.7", "@modelcontextprotocol/sdk": "^1.29.0", "@node-saml/passport-saml": "^5.1.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation-express": "^0.56.0", + "@opentelemetry/instrumentation-http": "^0.207.0", + "@opentelemetry/instrumentation-ioredis": "^0.55.0", + "@opentelemetry/instrumentation-mongodb": "^0.60.0", + "@opentelemetry/instrumentation-mongoose": "^0.54.0", + "@opentelemetry/instrumentation-undici": "^0.18.0", + "@opentelemetry/resources": "^2.6.1", + "@opentelemetry/sdk-node": "^0.207.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@smithy/node-http-handler": "^4.4.5", "ai-tokenizer": "^1.0.6", "axios": "^1.16.0", @@ -12866,7 +12876,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.1.tgz", "integrity": "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, @@ -14245,6 +14254,120 @@ "@opentelemetry/api": "^1.3.0" } }, + "node_modules/@opentelemetry/instrumentation-express": { + "version": "0.56.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.56.0.tgz", + "integrity": "sha512-rMV0WUTtAGEhHrHl3uDRIO97EkNUp4ewrW2iRVuP7kaV5qRT2b1pPV5PE75oR3GyLLSTooSAzGWl6CTm8eftKQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.207.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http": { + "version": "0.207.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.207.0.tgz", + "integrity": "sha512-FC4i5hVixTzuhg4SV2ycTEAYx+0E2hm+GwbdoVPSA6kna0pPVI4etzaA9UkpJ9ussumQheFXP6rkGIaFJjMxsw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/instrumentation": "0.207.0", + "@opentelemetry/semantic-conventions": "^1.29.0", + "forwarded-parse": "2.1.2" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", + "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation-ioredis": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.55.0.tgz", + "integrity": "sha512-ASuBMzh0ImmfOnWj9vCPtBMqSjr54/r/HluUIylwZB7xzTU6gL2SfybxySJMzEL9+386gJJVApwQktVznAtrWA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.207.0", + "@opentelemetry/redis-common": "^0.38.2" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mongodb": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.60.0.tgz", + "integrity": "sha512-8mKW2oyyWdYOKYpu70AGGAvLnExGlOoFT+Ylk9hJvWYHR5f6IrmVqwUMlzPM2WC4ihQ9crvqrtldMPmlVezTqg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.207.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mongoose": { + "version": "0.54.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.54.0.tgz", + "integrity": "sha512-R2JzMrxiz3R9m+cO6iVC85bma3E3vdBr06F+D4zGcEkJS9FaCRw6+Hdb+aQ1AaDRwEMTaw+F28+6VdMpniBOXQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.207.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-undici": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.18.0.tgz", + "integrity": "sha512-NalxLuZV621Xq4IhQkC+OXoZjAT8Xf6vYRdTjHitOXMU+4l/peRY05V7wGr4d7huf+vjyQry0XKlyhsEr4ouNw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.207.0", + "@opentelemetry/semantic-conventions": "^1.24.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.7.0" + } + }, "node_modules/@opentelemetry/instrumentation/node_modules/@opentelemetry/api-logs": { "version": "0.207.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.207.0.tgz", @@ -14504,12 +14627,20 @@ "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, + "node_modules/@opentelemetry/redis-common": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz", + "integrity": "sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + } + }, "node_modules/@opentelemetry/resources": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.6.1.tgz", "integrity": "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/semantic-conventions": "^1.29.0" @@ -28134,6 +28265,12 @@ "node": ">= 0.6" } }, + "node_modules/forwarded-parse": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", + "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", + "license": "MIT" + }, "node_modules/fraction.js": { "version": "5.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", @@ -28286,6 +28423,67 @@ "node": ">=14" } }, + "node_modules/gcp-metadata": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz", + "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/gcp-metadata/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/gcp-metadata/node_modules/gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/gcp-metadata/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/generic-names": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/generic-names/-/generic-names-4.0.0.tgz", @@ -28585,6 +28783,7 @@ "version": "6.1.1", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", "dependencies": { "gaxios": "^6.1.1", "google-logging-utils": "^0.0.2", @@ -28598,6 +28797,7 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", "engines": { "node": ">=14" } @@ -34726,6 +34926,52 @@ "node": ">=20.19.0" } }, + "node_modules/mongodb-memory-server-core/node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/gcp-metadata": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz", + "integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=14" + } + }, "node_modules/mongodb-memory-server-core/node_modules/mongodb": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.1.1.tgz", @@ -34787,6 +35033,27 @@ "node": ">=20.19.0" } }, + "node_modules/mongodb-memory-server-core/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/mongoose": { "version": "8.23.1", "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.23.1.tgz", @@ -44692,6 +44959,16 @@ "@librechat/agents": "^3.1.86", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.29.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation-express": "^0.56.0", + "@opentelemetry/instrumentation-http": "^0.207.0", + "@opentelemetry/instrumentation-ioredis": "^0.55.0", + "@opentelemetry/instrumentation-mongodb": "^0.60.0", + "@opentelemetry/instrumentation-mongoose": "^0.54.0", + "@opentelemetry/instrumentation-undici": "^0.18.0", + "@opentelemetry/resources": "^2.6.1", + "@opentelemetry/sdk-node": "^0.207.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@smithy/node-http-handler": "^4.4.5", "ai-tokenizer": "^1.0.6", "axios": "^1.16.0", diff --git a/packages/api/package.json b/packages/api/package.json index 1168c21bc6..048df30ba1 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -10,6 +10,10 @@ ".": { "require": "./dist/index.js", "types": "./dist/types/index.d.ts" + }, + "./telemetry": { + "require": "./dist/telemetry.js", + "types": "./dist/types/telemetry/index.d.ts" } }, "scripts": { @@ -103,6 +107,16 @@ "@librechat/agents": "^3.1.86", "@librechat/data-schemas": "*", "@modelcontextprotocol/sdk": "^1.29.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation-express": "^0.56.0", + "@opentelemetry/instrumentation-http": "^0.207.0", + "@opentelemetry/instrumentation-ioredis": "^0.55.0", + "@opentelemetry/instrumentation-mongodb": "^0.60.0", + "@opentelemetry/instrumentation-mongoose": "^0.54.0", + "@opentelemetry/instrumentation-undici": "^0.18.0", + "@opentelemetry/resources": "^2.6.1", + "@opentelemetry/sdk-node": "^0.207.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@smithy/node-http-handler": "^4.4.5", "ai-tokenizer": "^1.0.6", "axios": "^1.16.0", diff --git a/packages/api/rollup.config.js b/packages/api/rollup.config.js index 9a9de35e8f..becc482291 100644 --- a/packages/api/rollup.config.js +++ b/packages/api/rollup.config.js @@ -45,7 +45,10 @@ const plugins = [ ]; const cjsBuild = { - input: 'src/index.ts', + input: { + index: 'src/index.ts', + telemetry: 'src/telemetry/index.ts', + }, output: { dir: 'dist', format: 'cjs', diff --git a/packages/api/src/telemetry/config.spec.ts b/packages/api/src/telemetry/config.spec.ts new file mode 100644 index 0000000000..128f07a5a3 --- /dev/null +++ b/packages/api/src/telemetry/config.spec.ts @@ -0,0 +1,46 @@ +import { getTelemetryConfig } from './config'; + +describe('getTelemetryConfig', () => { + it('defaults tracing off', () => { + const config = getTelemetryConfig({}); + + expect(config.enabled).toBe(false); + expect(config.sdkDisabled).toBe(false); + expect(config.serviceName).toBe('librechat'); + expect(config.healthPath).toBe('/health'); + }); + + it('enables tracing only when OTEL_TRACING_ENABLED is true', () => { + expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'true' }).enabled).toBe(true); + expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'TRUE' }).enabled).toBe(true); + expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'false' }).enabled).toBe(false); + }); + + it('lets OTEL_SDK_DISABLED override tracing enablement', () => { + const config = getTelemetryConfig({ + OTEL_SDK_DISABLED: 'true', + OTEL_TRACING_ENABLED: 'true', + }); + + expect(config.enabled).toBe(false); + expect(config.sdkDisabled).toBe(true); + }); + + it('uses standard service env vars when provided', () => { + const config = getTelemetryConfig({ + OTEL_SERVICE_NAME: ' librechat-api ', + OTEL_SERVICE_VERSION: ' 1.2.3 ', + }); + + expect(config.serviceName).toBe('librechat-api'); + expect(config.serviceVersion).toBe('1.2.3'); + }); + + it('falls back to npm package version when service version is absent', () => { + const config = getTelemetryConfig({ + npm_package_version: '0.8.5', + }); + + expect(config.serviceVersion).toBe('0.8.5'); + }); +}); diff --git a/packages/api/src/telemetry/config.ts b/packages/api/src/telemetry/config.ts new file mode 100644 index 0000000000..340197aa65 --- /dev/null +++ b/packages/api/src/telemetry/config.ts @@ -0,0 +1,43 @@ +const DEFAULT_SERVICE_NAME = 'librechat'; +export const DEFAULT_HEALTH_PATH = '/health'; + +export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | 'stopped'; + +export interface TelemetryConfig { + enabled: boolean; + healthPath: string; + sdkDisabled: boolean; + serviceName: string; + serviceVersion?: string; +} + +function isTruthy(value?: string | boolean | null): boolean { + if (typeof value === 'boolean') { + return value; + } + if (typeof value === 'string') { + return value.trim().toLowerCase() === 'true'; + } + return false; +} + +function normalizeEnvValue(value?: string): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): TelemetryConfig { + const sdkDisabled = isTruthy(env.OTEL_SDK_DISABLED); + const enabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled; + const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME; + const serviceVersion = + normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version); + + return { + enabled, + serviceName, + sdkDisabled, + serviceVersion, + healthPath: DEFAULT_HEALTH_PATH, + }; +} diff --git a/packages/api/src/telemetry/index.ts b/packages/api/src/telemetry/index.ts new file mode 100644 index 0000000000..2eb188bf84 --- /dev/null +++ b/packages/api/src/telemetry/index.ts @@ -0,0 +1,5 @@ +export { getTelemetryConfig } from './config'; +export { initializeTelemetry, shutdownTelemetry } from './sdk'; +export { telemetryErrorMiddleware, telemetryMiddleware } from './middleware'; +export type { TelemetryConfig, TelemetryStatus } from './config'; +export type { TelemetryController } from './sdk'; diff --git a/packages/api/src/telemetry/middleware.spec.ts b/packages/api/src/telemetry/middleware.spec.ts new file mode 100644 index 0000000000..271d8c0a11 --- /dev/null +++ b/packages/api/src/telemetry/middleware.spec.ts @@ -0,0 +1,395 @@ +import { EventEmitter } from 'node:events'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; +import type { NextFunction, Response } from 'express'; +import type { Span } from '@opentelemetry/api'; +import type { ServerRequest } from '~/types'; +import { getTelemetryRequestSpan } from './sdk'; +import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware'; + +jest.mock('./sdk', () => ({ + getTelemetryRequestSpan: jest.fn(), +})); + +const mockGetTelemetryRequestSpan = getTelemetryRequestSpan as jest.MockedFunction< + typeof getTelemetryRequestSpan +>; + +interface MockResponse extends EventEmitter { + statusCode: number; + writableEnded: boolean; +} + +function createSpan(): jest.Mocked { + const span = {} as jest.Mocked; + span.addEvent = jest.fn, Parameters>(() => span); + span.addLink = jest.fn, Parameters>(() => span); + span.addLinks = jest.fn, Parameters>(() => span); + span.end = jest.fn>(); + span.isRecording = jest.fn>(() => true); + span.recordException = jest.fn>(); + span.setAttribute = jest.fn, Parameters>(() => span); + span.setAttributes = jest.fn, Parameters>(() => span); + span.setStatus = jest.fn, Parameters>(() => span); + span.spanContext = jest.fn, Parameters>( + () => ({ + spanId: '0000000000000000', + traceFlags: 0, + traceId: '00000000000000000000000000000000', + }), + ); + span.updateName = jest.fn, Parameters>(() => span); + return span; +} + +function createResponse(statusCode = 200): MockResponse { + const res = new EventEmitter() as MockResponse; + res.statusCode = statusCode; + res.writableEnded = false; + return res; +} + +function createRequest(overrides: Partial = {}): ServerRequest { + return { + baseUrl: '/api/messages', + body: { + prompt: 'do not capture this prompt', + text: 'do not capture this body', + }, + headers: { + authorization: 'Bearer do-not-capture-this-auth-header', + cookie: 'session=do-not-capture-this-cookie', + 'x-api-key': 'do-not-capture-this-api-key', + }, + method: 'POST', + path: '/api/messages/conversation-1', + route: { path: '/:conversationId' }, + user: { + email: 'do-not-capture@example.com', + id: 'user-1', + tenantId: 'tenant-1', + } as ServerRequest['user'], + ...overrides, + } as ServerRequest; +} + +afterEach(() => { + mockGetTelemetryRequestSpan.mockReset(); +}); + +describe('telemetryMiddleware', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('passes through without an active span', () => { + const next = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined); + + telemetryMiddleware(createRequest(), createResponse() as Response, next); + + expect(next).toHaveBeenCalledTimes(1); + }); + + it('uses the stored request span for deferred completion attributes', () => { + const activeSpan = createSpan(); + const requestSpan = createSpan(); + const req = createRequest(); + const res = createResponse(202); + const next: NextFunction = jest.fn(); + mockGetTelemetryRequestSpan.mockReturnValue(requestSpan); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan); + + telemetryMiddleware(req, res as Response, next); + res.emit('finish'); + + expect(next).toHaveBeenCalledTimes(1); + expect(trace.getActiveSpan).not.toHaveBeenCalled(); + expect(activeSpan.setAttributes).not.toHaveBeenCalled(); + expect(requestSpan.setAttributes).toHaveBeenCalledWith({ + 'http.request.method': 'POST', + }); + expect(requestSpan.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'http.response.status_code': 202, + 'http.route': '/api/messages/:conversationId', + }), + ); + }); + + it('records safe route and identity attributes without body content', () => { + const span = createSpan(); + const req = createRequest(); + const res = createResponse(201); + const next: NextFunction = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(req, res as Response, next); + res.emit('finish'); + + expect(next).toHaveBeenCalledTimes(1); + expect(span.setAttributes).toHaveBeenCalledWith({ + 'http.request.method': 'POST', + }); + expect(span.setAttributes).toHaveBeenCalledWith({ + 'enduser.id': 'user-1', + 'librechat.tenant.id': 'tenant-1', + }); + expect(span.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'http.response.status_code': 201, + 'http.route': '/api/messages/:conversationId', + }), + ); + + const capturedAttributes = JSON.stringify(span.setAttributes.mock.calls); + expect(capturedAttributes).not.toContain('do not capture this prompt'); + expect(capturedAttributes).not.toContain('do not capture this body'); + expect(capturedAttributes).not.toContain('do-not-capture@example.com'); + expect(capturedAttributes).not.toContain('conversation-1'); + expect(capturedAttributes).not.toContain('do-not-capture-this-auth-header'); + expect(capturedAttributes).not.toContain('do-not-capture-this-cookie'); + expect(capturedAttributes).not.toContain('do-not-capture-this-api-key'); + }); + + it('records identity attributes populated by downstream middleware', () => { + const span = createSpan(); + const req = createRequest({ + headers: {}, + user: undefined, + }); + const res = createResponse(200); + const next: NextFunction = jest.fn(() => { + req.user = { + id: 'late-user', + tenantId: 'late-tenant', + } as ServerRequest['user']; + }); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(req, res as Response, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(span.setAttributes).toHaveBeenCalledTimes(1); + + res.emit('finish'); + + expect(span.setAttributes).toHaveBeenCalledWith({ + 'enduser.id': 'late-user', + 'librechat.tenant.id': 'late-tenant', + }); + }); + + it('does not derive tenant identity from request headers', () => { + const span = createSpan(); + const req = createRequest({ + headers: { + 'x-tenant-id': 'spoofed-tenant', + }, + user: undefined, + }); + const res = createResponse(200); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(req, res as Response, jest.fn()); + res.emit('finish'); + + expect(span.setAttributes).not.toHaveBeenCalledWith( + expect.objectContaining({ + 'librechat.tenant.id': 'spoofed-tenant', + }), + ); + expect(JSON.stringify(span.setAttributes.mock.calls)).not.toContain('spoofed-tenant'); + }); + + it('ignores health checks', () => { + const span = createSpan(); + const next = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware( + createRequest({ + baseUrl: '', + path: '/health', + route: undefined, + }), + createResponse() as Response, + next, + ); + + expect(next).toHaveBeenCalledTimes(1); + expect(span.setAttributes).not.toHaveBeenCalled(); + }); + + it('uses a low-cardinality fallback for unmatched API routes', () => { + const span = createSpan(); + const req = createRequest({ + baseUrl: '', + path: '/api/nonexistent/123', + route: undefined, + }); + const res = createResponse(404); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(req, res as Response, jest.fn()); + res.emit('finish'); + + expect(span.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'http.route': '/api/*', + 'http.response.status_code': 404, + }), + ); + }); + + it('uses a low-cardinality fallback for unmatched SPA routes', () => { + const span = createSpan(); + const req = createRequest({ + baseUrl: '', + path: '/chat/conversation-id', + route: undefined, + }); + const res = createResponse(200); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(req, res as Response, jest.fn()); + res.emit('finish'); + + expect(span.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'http.route': 'spa_fallback', + 'http.response.status_code': 200, + }), + ); + }); + + it('marks server responses as errored', () => { + const span = createSpan(); + const req = createRequest(); + const res = createResponse(500); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(req, res as Response, jest.fn()); + res.emit('finish'); + + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + }); + + it('records completion attributes only once when finish and close both fire', () => { + const span = createSpan(); + const res = createResponse(200); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(createRequest(), res as Response, jest.fn()); + res.emit('finish'); + res.emit('close'); + + expect(span.setAttributes).toHaveBeenCalledTimes(3); + }); + + it('marks client disconnects before finish as aborted errors', () => { + const span = createSpan(); + const res = createResponse(200); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryMiddleware(createRequest(), res as Response, jest.fn()); + res.emit('close'); + + expect(span.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'http.response.status_code': 499, + 'librechat.request.aborted': true, + }), + ); + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + }); +}); + +describe('telemetryErrorMiddleware', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('records exceptions and forwards the error', () => { + const span = createSpan(); + const error = new TypeError('boom'); + const next = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next); + + expect(span.recordException).toHaveBeenCalledWith(error); + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + expect(span.setAttributes).toHaveBeenCalledWith({ + 'enduser.id': 'user-1', + 'librechat.tenant.id': 'tenant-1', + }); + expect(span.setAttributes).toHaveBeenCalledWith({ + 'error.type': 'TypeError', + 'http.route': '/api/messages/:conversationId', + }); + expect(next).toHaveBeenCalledWith(error); + }); + + it('records exceptions on the stored request span when available', () => { + const activeSpan = createSpan(); + const requestSpan = createSpan(); + const error = new TypeError('boom'); + const next = jest.fn(); + mockGetTelemetryRequestSpan.mockReturnValue(requestSpan); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan); + + telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next); + + expect(trace.getActiveSpan).not.toHaveBeenCalled(); + expect(activeSpan.recordException).not.toHaveBeenCalled(); + expect(requestSpan.recordException).toHaveBeenCalledWith(error); + expect(requestSpan.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + expect(next).toHaveBeenCalledWith(error); + }); + + it('handles non-Error values without throwing', () => { + const span = createSpan(); + const next = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryErrorMiddleware('boom', createRequest(), createResponse() as Response, next); + + expect(span.recordException).toHaveBeenCalledWith('boom'); + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + expect(span.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'error.type': 'string', + 'http.route': '/api/messages/:conversationId', + }), + ); + expect(next).toHaveBeenCalledWith('boom'); + }); + + it('handles null error values without throwing', () => { + const span = createSpan(); + const next = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span); + + telemetryErrorMiddleware(null, createRequest(), createResponse() as Response, next); + + expect(span.recordException).not.toHaveBeenCalled(); + expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); + expect(span.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + 'error.type': 'null', + 'http.route': '/api/messages/:conversationId', + }), + ); + expect(next).toHaveBeenCalledWith(null); + }); + + it('forwards the error without an active span', () => { + const error = new Error('boom'); + const next = jest.fn(); + jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined); + + telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next); + + expect(next).toHaveBeenCalledWith(error); + }); +}); diff --git a/packages/api/src/telemetry/middleware.ts b/packages/api/src/telemetry/middleware.ts new file mode 100644 index 0000000000..d9c110c1b9 --- /dev/null +++ b/packages/api/src/telemetry/middleware.ts @@ -0,0 +1,171 @@ +import { SpanStatusCode, trace } from '@opentelemetry/api'; +import type { Span, Attributes } from '@opentelemetry/api'; +import type { NextFunction, Response } from 'express'; +import type { ServerRequest } from '~/types'; +import { getTelemetryRequestSpan } from './sdk'; +import { DEFAULT_HEALTH_PATH } from './config'; + +const CLIENT_CLOSED_REQUEST_STATUS_CODE = 499; + +type ExpressErrorValue = + | Error + | string + | number + | boolean + | bigint + | symbol + | object + | null + | undefined; + +function getUserId(req: ServerRequest): string | undefined { + return req.user?.id; +} + +function getTenantId(req: ServerRequest): string | undefined { + return req.user?.tenantId; +} + +function isHealthPath(req: ServerRequest): boolean { + return req.path === DEFAULT_HEALTH_PATH; +} + +function isApiPath(req: ServerRequest): boolean { + return req.path === '/api' || req.path.startsWith('/api/'); +} + +function getRoutePath(req: ServerRequest): string { + const routePath = req.route?.path; + if (typeof routePath === 'string') { + return `${req.baseUrl}${routePath}`; + } + + if (isHealthPath(req)) { + return '/health'; + } + + if (isApiPath(req)) { + return '/api/*'; + } + + return 'spa_fallback'; +} + +function setIdentityAttributes(span: Span, req: ServerRequest): void { + const userId = getUserId(req); + const tenantId = getTenantId(req); + + if (!userId && !tenantId) { + return; + } + + const attributes: Attributes = {}; + + if (userId) { + attributes['enduser.id'] = userId; + } + + if (tenantId) { + attributes['librechat.tenant.id'] = tenantId; + } + + span.setAttributes(attributes); +} + +function setCompletionAttributes( + span: Span, + req: ServerRequest, + res: Response, + aborted = false, +): void { + const statusCode = aborted ? CLIENT_CLOSED_REQUEST_STATUS_CODE : res.statusCode; + const routePath = getRoutePath(req); + const attributes: Attributes = { + 'http.route': routePath, + 'http.response.status_code': statusCode, + }; + + if (aborted) { + attributes['librechat.request.aborted'] = true; + } + + setIdentityAttributes(span, req); + span.setAttributes(attributes); + + if (aborted || statusCode >= 500) { + span.setStatus({ code: SpanStatusCode.ERROR }); + } +} + +export function telemetryMiddleware(req: ServerRequest, res: Response, next: NextFunction): void { + if (isHealthPath(req)) { + next(); + return; + } + + const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan(); + if (!span) { + next(); + return; + } + + span.setAttributes({ + 'http.request.method': req.method, + }); + + let completed = false; + const complete = () => { + if (completed) { + return; + } + completed = true; + setCompletionAttributes(span, req, res); + }; + + const close = () => { + if (completed) { + return; + } + completed = true; + setCompletionAttributes(span, req, res, !res.writableEnded); + }; + + res.once('finish', complete); + res.once('close', close); + next(); +} + +export function telemetryErrorMiddleware( + err: ExpressErrorValue, + req: ServerRequest, + _res: Response, + next: NextFunction, +): void { + const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan(); + if (span) { + const routePath = getRoutePath(req); + if (err) { + span.recordException(err instanceof Error ? err : String(err)); + } + span.setStatus({ code: SpanStatusCode.ERROR }); + setIdentityAttributes(span, req); + span.setAttributes({ + 'error.type': getErrorType(err), + 'http.route': routePath, + }); + } + + next(err); +} + +function getErrorType(err: ExpressErrorValue): string { + if (err instanceof Error) { + return err.name || err.constructor.name; + } + + if (err === null) { + return 'null'; + } + + return typeof err; +} diff --git a/packages/api/src/telemetry/sdk.spec.ts b/packages/api/src/telemetry/sdk.spec.ts new file mode 100644 index 0000000000..548ecc6f2a --- /dev/null +++ b/packages/api/src/telemetry/sdk.spec.ts @@ -0,0 +1,357 @@ +import { Socket } from 'node:net'; +import { IncomingMessage } from 'node:http'; +import type { Span } from '@opentelemetry/api'; + +interface HttpInstrumentationOptions { + requestHook?: (span: Span, request: object) => void; + startIncomingSpanHook?: (request: IncomingMessage) => Record; +} + +const mockStart = jest.fn(); +const mockShutdown = jest.fn(); +const mockNodeSDK = jest.fn(() => ({ + start: mockStart, + shutdown: mockShutdown, +})); +const mockExpressInstrumentation = jest.fn(() => ({ name: 'express' })); +const mockHttpInstrumentation = jest.fn((options?: HttpInstrumentationOptions) => ({ + name: 'http', + options, +})); +const mockIORedisInstrumentation = jest.fn(() => ({ name: 'ioredis' })); +const mockMongoDBInstrumentation = jest.fn(() => ({ name: 'mongodb' })); +const mockMongooseInstrumentation = jest.fn(() => ({ name: 'mongoose' })); +const mockUndiciInstrumentation = jest.fn(() => ({ name: 'undici' })); +const mockResourceFromAttributes = jest.fn((attributes: object) => ({ attributes })); + +jest.mock( + '@opentelemetry/sdk-node', + () => ({ + NodeSDK: mockNodeSDK, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/instrumentation-express', + () => ({ + ExpressInstrumentation: mockExpressInstrumentation, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/instrumentation-http', + () => ({ + HttpInstrumentation: mockHttpInstrumentation, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/instrumentation-ioredis', + () => ({ + IORedisInstrumentation: mockIORedisInstrumentation, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/instrumentation-mongodb', + () => ({ + MongoDBInstrumentation: mockMongoDBInstrumentation, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/instrumentation-mongoose', + () => ({ + MongooseInstrumentation: mockMongooseInstrumentation, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/instrumentation-undici', + () => ({ + UndiciInstrumentation: mockUndiciInstrumentation, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/resources', + () => ({ + resourceFromAttributes: mockResourceFromAttributes, + }), + { virtual: true }, +); + +jest.mock( + '@opentelemetry/semantic-conventions', + () => ({ + ATTR_SERVICE_NAME: 'service.name', + ATTR_SERVICE_VERSION: 'service.version', + }), + { virtual: true }, +); + +async function flushSignalShutdown(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + +describe('telemetry SDK lifecycle', () => { + let emitWarningSpy: jest.SpyInstance; + let getTelemetryRequestSpan: (typeof import('./sdk'))['getTelemetryRequestSpan']; + let initializeTelemetry: (typeof import('./sdk'))['initializeTelemetry']; + let resetTelemetryForTests: (typeof import('./sdk'))['resetTelemetryForTests']; + let shutdownTelemetry: (typeof import('./sdk'))['shutdownTelemetry']; + + beforeEach(async () => { + jest.clearAllMocks(); + ({ getTelemetryRequestSpan, initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } = + await import('./sdk')); + await resetTelemetryForTests(); + Reflect.deleteProperty(globalThis, 'Bun'); + emitWarningSpy = jest.spyOn(process, 'emitWarning').mockImplementation(() => true); + }); + + afterEach(async () => { + await resetTelemetryForTests(); + emitWarningSpy.mockRestore(); + Reflect.deleteProperty(globalThis, 'Bun'); + }); + + it('does not initialize when tracing is disabled by default', () => { + const controller = initializeTelemetry({}); + + expect(controller.enabled).toBe(false); + expect(controller.status).toBe('disabled'); + expect(mockNodeSDK).not.toHaveBeenCalled(); + }); + + it('does not initialize when OTEL_SDK_DISABLED is true', () => { + const controller = initializeTelemetry({ + OTEL_SDK_DISABLED: 'true', + OTEL_TRACING_ENABLED: 'true', + }); + + expect(controller.enabled).toBe(false); + expect(controller.status).toBe('disabled'); + expect(mockNodeSDK).not.toHaveBeenCalled(); + }); + + it('does not initialize under Bun runtime', () => { + Object.defineProperty(globalThis, 'Bun', { + configurable: true, + value: {}, + }); + + const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + expect(controller.enabled).toBe(false); + expect(controller.status).toBe('disabled'); + expect(mockNodeSDK).not.toHaveBeenCalled(); + }); + + it('starts the Node SDK once when enabled', () => { + const first = initializeTelemetry({ + OTEL_SERVICE_NAME: 'librechat-test', + OTEL_TRACING_ENABLED: 'true', + }); + const second = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + expect(first.enabled).toBe(true); + expect(first.status).toBe('started'); + expect(second.enabled).toBe(true); + expect(mockNodeSDK).toHaveBeenCalledTimes(1); + expect(mockStart).toHaveBeenCalledTimes(1); + expect(mockResourceFromAttributes).toHaveBeenCalledWith({ + 'service.name': 'librechat-test', + }); + expect(mockHttpInstrumentation).toHaveBeenCalledWith( + expect.objectContaining({ + headersToSpanAttributes: { + client: { requestHeaders: [], responseHeaders: [] }, + server: { requestHeaders: [], responseHeaders: [] }, + }, + }), + ); + expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1); + expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1); + expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1); + expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1); + expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1); + }); + + it('tracks HTTP server request spans for completion updates', () => { + const span = {} as Span; + const request = new IncomingMessage(new Socket()); + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0]; + const requestHook = instrumentationOptions?.requestHook; + + if (!requestHook) { + throw new Error('HTTP instrumentation requestHook was not configured'); + } + + requestHook(span, request); + + expect(getTelemetryRequestSpan(request)).toBe(span); + }); + + it('redacts incoming URL attributes before HTTP spans are exported', () => { + const request = new IncomingMessage(new Socket()); + request.url = '/oauth/callback?code=secret-code&state=secret-state'; + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0]; + const startIncomingSpanHook = instrumentationOptions?.startIncomingSpanHook; + + if (!startIncomingSpanHook) { + throw new Error('HTTP instrumentation startIncomingSpanHook was not configured'); + } + + const attributes = startIncomingSpanHook(request); + + expect(attributes).toEqual({ + 'http.target': 'spa_fallback?[REDACTED]', + 'http.url': 'spa_fallback?[REDACTED]', + 'url.full': 'spa_fallback?[REDACTED]', + 'url.path': 'spa_fallback', + 'url.query': '[REDACTED]', + }); + expect(JSON.stringify(attributes)).not.toContain('secret-code'); + expect(JSON.stringify(attributes)).not.toContain('secret-state'); + }); + + it('reflects lifecycle status from the controller getter', async () => { + const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + expect(controller.status).toBe('started'); + await controller.shutdown(); + expect(controller.status).toBe('stopped'); + expect(controller.enabled).toBe(false); + }); + + it('handles async SDK start failures without throwing', async () => { + mockStart.mockRejectedValueOnce(new Error('async start failed')); + + const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + expect(controller.enabled).toBe(true); + expect(controller.status).toBe('starting'); + await controller.shutdown(); + expect(controller.enabled).toBe(false); + expect(controller.status).toBe('failed'); + expect(emitWarningSpy).toHaveBeenCalledWith( + 'OpenTelemetry initialization failed: async start failed', + { code: 'LIBRECHAT_OTEL' }, + ); + }); + + it('returns failed status without throwing when SDK start fails', () => { + mockStart.mockImplementationOnce(() => { + throw new Error('start failed'); + }); + + const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + expect(controller.enabled).toBe(false); + expect(controller.status).toBe('failed'); + expect(emitWarningSpy).toHaveBeenCalledWith( + 'OpenTelemetry initialization failed: start failed', + { code: 'LIBRECHAT_OTEL' }, + ); + }); + + it('shuts down the active SDK idempotently', async () => { + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + await shutdownTelemetry(); + await shutdownTelemetry(); + + expect(mockShutdown).toHaveBeenCalledTimes(1); + }); + + it('coalesces concurrent shutdown calls', async () => { + let resolveShutdown: () => void = () => undefined; + mockShutdown.mockReturnValueOnce( + new Promise((resolve) => { + resolveShutdown = resolve; + }), + ); + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + const firstShutdown = shutdownTelemetry(); + const secondShutdown = shutdownTelemetry(); + expect(mockShutdown).toHaveBeenCalledTimes(1); + + resolveShutdown(); + await Promise.all([firstShutdown, secondShutdown]); + }); + + it('keeps the active SDK available when shutdown fails', async () => { + mockShutdown.mockRejectedValueOnce(new Error('shutdown failed')); + const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + await expect(shutdownTelemetry()).rejects.toThrow('shutdown failed'); + expect(controller.status).toBe('started'); + + await shutdownTelemetry(); + expect(mockShutdown).toHaveBeenCalledTimes(2); + expect(controller.status).toBe('stopped'); + }); + + it.each(['SIGTERM', 'SIGINT'])( + 'does not force process exit when another %s handler is registered', + async (signal) => { + const otherHandler = jest.fn(); + const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true); + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + process.once(signal, otherHandler); + + process.emit(signal, signal); + await flushSignalShutdown(); + + expect(mockShutdown).toHaveBeenCalledTimes(1); + expect(otherHandler).toHaveBeenCalledTimes(1); + expect(killSpy).not.toHaveBeenCalled(); + + killSpy.mockRestore(); + }, + ); + + it.each(['SIGTERM', 'SIGINT'])( + 'reraises the shutdown %s signal when telemetry is the only signal handler', + async (signal) => { + const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true); + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + process.emit(signal, signal); + await flushSignalShutdown(); + + expect(mockShutdown).toHaveBeenCalledTimes(1); + expect(killSpy).toHaveBeenCalledWith(process.pid, signal); + + killSpy.mockRestore(); + }, + ); + + it('warns and reraises the signal when shutdown rejects', async () => { + mockShutdown.mockRejectedValueOnce(new Error('signal shutdown failed')); + const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true); + initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' }); + + process.emit('SIGTERM', 'SIGTERM'); + await flushSignalShutdown(); + + expect(mockShutdown).toHaveBeenCalledTimes(1); + expect(emitWarningSpy).toHaveBeenCalledWith( + 'OpenTelemetry shutdown failed: signal shutdown failed', + { code: 'LIBRECHAT_OTEL' }, + ); + expect(killSpy).toHaveBeenCalledWith(process.pid, 'SIGTERM'); + + killSpy.mockRestore(); + }); +}); diff --git a/packages/api/src/telemetry/sdk.ts b/packages/api/src/telemetry/sdk.ts new file mode 100644 index 0000000000..49bd8a7ffc --- /dev/null +++ b/packages/api/src/telemetry/sdk.ts @@ -0,0 +1,328 @@ +import { IncomingMessage } from 'node:http'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici'; +import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; +import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'; +import { MongoDBInstrumentation } from '@opentelemetry/instrumentation-mongodb'; +import { MongooseInstrumentation } from '@opentelemetry/instrumentation-mongoose'; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; +import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node'; +import type { Span, Attributes } from '@opentelemetry/api'; +import type { TelemetryConfig, TelemetryStatus } from './config'; +import { getTelemetryConfig } from './config'; + +export interface TelemetryController { + readonly enabled: boolean; + readonly status: TelemetryStatus; + shutdown: () => Promise; +} + +const WARNING_CODE = 'LIBRECHAT_OTEL'; +const REDACTED_QUERY_VALUE = '[REDACTED]'; +const SIGNAL_SHUTDOWN_TIMEOUT_MS = 5_000; + +interface RegisteredSignal { + signal: NodeJS.Signals; + listener: NodeJS.SignalsListener; +} + +let activeSdk: NodeSDK | undefined; +let pendingSdk: NodeSDK | undefined; +let startPromise: Promise | undefined; +let shutdownPromise: Promise | undefined; +let status: TelemetryStatus = 'stopped'; +let registeredSignals: RegisteredSignal[] = []; +let requestSpans = new WeakMap(); + +function isBunRuntime(): boolean { + return Reflect.get(globalThis, 'Bun') != null; +} + +function shouldIgnoreIncomingRequest(request: IncomingMessage, healthPath: string): boolean { + return request.url === healthPath || request.url?.startsWith(`${healthPath}?`) === true; +} + +function getIncomingUrlInfo(request: IncomingMessage): { hasQuery: boolean; pathname: string } { + const rawUrl = request.url ?? '/'; + + try { + const parsedUrl = new URL(rawUrl, 'http://localhost'); + return { + hasQuery: parsedUrl.search.length > 1, + pathname: parsedUrl.pathname || '/', + }; + } catch { + const queryIndex = rawUrl.indexOf('?'); + return { + hasQuery: queryIndex >= 0 && queryIndex < rawUrl.length - 1, + pathname: queryIndex >= 0 ? rawUrl.slice(0, queryIndex) || '/' : rawUrl || '/', + }; + } +} + +function getLowCardinalityUrlPath(pathname: string, healthPath: string): string { + if (pathname === healthPath) { + return healthPath; + } + + if (pathname === '/api' || pathname.startsWith('/api/')) { + return '/api/*'; + } + + return 'spa_fallback'; +} + +function getSanitizedIncomingUrlAttributes( + request: IncomingMessage, + healthPath: string, +): Attributes { + const { hasQuery, pathname } = getIncomingUrlInfo(request); + const safePath = getLowCardinalityUrlPath(pathname, healthPath); + const safeTarget = hasQuery ? `${safePath}?${REDACTED_QUERY_VALUE}` : safePath; + const attributes: Attributes = { + 'http.target': safeTarget, + 'http.url': safeTarget, + 'url.full': safeTarget, + 'url.path': safePath, + }; + + if (hasQuery) { + attributes['url.query'] = REDACTED_QUERY_VALUE; + } + + return attributes; +} + +function getResourceAttributes(config: TelemetryConfig): Attributes { + const attributes: Attributes = { + [ATTR_SERVICE_NAME]: config.serviceName, + }; + + if (config.serviceVersion) { + attributes[ATTR_SERVICE_VERSION] = config.serviceVersion; + } + + return attributes; +} + +function createSdk(config: TelemetryConfig): NodeSDK { + const sdkConfig: Partial = { + resource: resourceFromAttributes(getResourceAttributes(config)), + instrumentations: [ + new HttpInstrumentation({ + headersToSpanAttributes: { + client: { requestHeaders: [], responseHeaders: [] }, + server: { requestHeaders: [], responseHeaders: [] }, + }, + requestHook: (span: Span, request: object) => { + if (request instanceof IncomingMessage) { + requestSpans.set(request, span); + } + }, + startIncomingSpanHook: (request: IncomingMessage) => + getSanitizedIncomingUrlAttributes(request, config.healthPath), + ignoreIncomingRequestHook: (request: IncomingMessage) => + shouldIgnoreIncomingRequest(request, config.healthPath), + }), + new ExpressInstrumentation(), + new MongoDBInstrumentation(), + new MongooseInstrumentation(), + new IORedisInstrumentation(), + new UndiciInstrumentation(), + ], + }; + + return new NodeSDK(sdkConfig); +} + +export function getTelemetryRequestSpan(request: IncomingMessage): Span | undefined { + return requestSpans.get(request); +} + +/** + * NodeSDK.start has been synchronous in some supported OpenTelemetry versions + * and promise-returning in others, so the lifecycle wrapper accepts either form. + */ +function startSdk(sdk: NodeSDK): void | Promise { + return (sdk as NodeSDK & { start: () => void | Promise }).start(); +} + +function emitWarning(message: string): void { + process.emitWarning(message, { code: WARNING_CODE }); +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isControllerEnabled(): boolean { + return status === 'starting' || status === 'started'; +} + +function makeController(): TelemetryController { + return { + get enabled() { + return isControllerEnabled(); + }, + get status() { + return status; + }, + shutdown: shutdownTelemetry, + }; +} + +function unregisterShutdownHandlers(): void { + for (const { signal, listener } of registeredSignals) { + process.removeListener(signal, listener); + } + registeredSignals = []; +} + +function registerShutdownHandlers(): void { + if (registeredSignals.length > 0) { + return; + } + + const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT']; + registeredSignals = signals.map((signal) => { + const listener: NodeJS.SignalsListener = () => { + const shouldReraiseSignal = process.listenerCount(signal) === 0; + withTimeout(shutdownTelemetry(), SIGNAL_SHUTDOWN_TIMEOUT_MS) + .catch((error) => { + emitWarning(`OpenTelemetry shutdown failed: ${getErrorMessage(error)}`); + }) + .finally(() => { + if (shouldReraiseSignal) { + process.kill(process.pid, signal); + } + }); + }; + process.once(signal, listener); + return { signal, listener }; + }); +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + let timeout: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { + reject(new Error(`timed out after ${timeoutMs}ms`)); + }, timeoutMs); + timeout.unref?.(); + }); + + return Promise.race([promise, timeoutPromise]).finally(() => { + if (timeout) { + clearTimeout(timeout); + } + }); +} + +export function initializeTelemetry(env: NodeJS.ProcessEnv = process.env): TelemetryController { + if (activeSdk || pendingSdk) { + return makeController(); + } + + const config = getTelemetryConfig(env); + if (!config.enabled || isBunRuntime()) { + status = 'disabled'; + return makeController(); + } + + try { + const sdk = createSdk(config); + const result = startSdk(sdk); + if (result) { + pendingSdk = sdk; + status = 'starting'; + const pendingStart = result + .then(() => { + if (pendingSdk === sdk) { + pendingSdk = undefined; + activeSdk = sdk; + status = 'started'; + registerShutdownHandlers(); + } + }) + .catch((error) => { + if (pendingSdk === sdk) { + pendingSdk = undefined; + status = 'failed'; + emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`); + } + }); + startPromise = pendingStart; + void pendingStart.finally(() => { + if (startPromise === pendingStart) { + startPromise = undefined; + } + }); + return makeController(); + } + + activeSdk = sdk; + status = 'started'; + registerShutdownHandlers(); + return makeController(); + } catch (error) { + status = 'failed'; + emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`); + return makeController(); + } +} + +async function performShutdownTelemetry(): Promise { + if (startPromise) { + await startPromise; + } + + if (!activeSdk) { + status = status === 'started' ? 'stopped' : status; + return; + } + + const sdk = activeSdk; + try { + await sdk.shutdown(); + activeSdk = undefined; + status = 'stopped'; + unregisterShutdownHandlers(); + } catch (error) { + status = 'started'; + throw error; + } +} + +export function shutdownTelemetry(): Promise { + if (!shutdownPromise) { + shutdownPromise = performShutdownTelemetry().finally(() => { + shutdownPromise = undefined; + }); + } + + return shutdownPromise; +} + +export async function resetTelemetryForTests(): Promise { + try { + if (startPromise) { + await startPromise.catch(() => undefined); + } + + if (shutdownPromise) { + await shutdownPromise.catch(() => undefined); + } else if (activeSdk) { + await Promise.resolve(activeSdk.shutdown()).catch(() => undefined); + } + } finally { + activeSdk = undefined; + pendingSdk = undefined; + startPromise = undefined; + shutdownPromise = undefined; + status = 'stopped'; + requestSpans = new WeakMap(); + unregisterShutdownHandlers(); + } +}