mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-06-10 01:44:44 +00:00
🛰️ fix: Redact Outbound Telemetry URL Queries (#13133)
* fix: redact outbound telemetry URL queries * fix: handle telemetry redaction edge cases * fix: keep sanitized telemetry URLs absolute * fix: infer https telemetry URL scheme * fix: avoid port-only telemetry protocol inference * fix: bracket ipv6 telemetry hosts
This commit is contained in:
Danny Avila
parent
5b11a5a076
commit
27266bbcdc
2 changed files with 391 additions and 2 deletions
|
|
@ -1,10 +1,17 @@
|
|||
import { Socket } from 'node:net';
|
||||
import { IncomingMessage } from 'node:http';
|
||||
import { Agent as HttpsAgent } from 'node:https';
|
||||
import type { Span } from '@opentelemetry/api';
|
||||
import type { RequestOptions } from 'node:http';
|
||||
|
||||
interface HttpInstrumentationOptions {
|
||||
requestHook?: (span: Span, request: object) => void;
|
||||
startIncomingSpanHook?: (request: IncomingMessage) => Record<string, string>;
|
||||
startOutgoingSpanHook?: (request: RequestOptions) => Record<string, string>;
|
||||
}
|
||||
|
||||
interface UndiciInstrumentationOptions {
|
||||
startSpanHook?: (request: { origin?: string; path?: string }) => Record<string, string>;
|
||||
}
|
||||
|
||||
const mockStart = jest.fn();
|
||||
|
|
@ -21,7 +28,10 @@ const mockHttpInstrumentation = jest.fn((options?: HttpInstrumentationOptions) =
|
|||
const mockIORedisInstrumentation = jest.fn(() => ({ name: 'ioredis' }));
|
||||
const mockMongoDBInstrumentation = jest.fn(() => ({ name: 'mongodb' }));
|
||||
const mockMongooseInstrumentation = jest.fn(() => ({ name: 'mongoose' }));
|
||||
const mockUndiciInstrumentation = jest.fn(() => ({ name: 'undici' }));
|
||||
const mockUndiciInstrumentation = jest.fn((options?: UndiciInstrumentationOptions) => ({
|
||||
name: 'undici',
|
||||
options,
|
||||
}));
|
||||
const mockResourceFromAttributes = jest.fn((attributes: object) => ({ attributes }));
|
||||
|
||||
jest.mock(
|
||||
|
|
@ -225,6 +235,188 @@ describe('telemetry SDK lifecycle', () => {
|
|||
expect(JSON.stringify(attributes)).not.toContain('secret-state');
|
||||
});
|
||||
|
||||
it('redacts outgoing HTTP URL query attributes before client spans are exported', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startOutgoingSpanHook = instrumentationOptions?.startOutgoingSpanHook;
|
||||
|
||||
if (!startOutgoingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startOutgoingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startOutgoingSpanHook({
|
||||
protocol: 'http:',
|
||||
hostname: '127.0.0.1',
|
||||
port: 33169,
|
||||
path: '/custom-action?api_key=LC_ACTION_QUERY_SECRET_67890&user_text=sensitive+prompt+words',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/custom-action?api_key=[REDACTED]&user_text=[REDACTED]',
|
||||
'http.url': 'http://127.0.0.1:33169/custom-action?api_key=[REDACTED]&user_text=[REDACTED]',
|
||||
'url.full': 'http://127.0.0.1:33169/custom-action?api_key=[REDACTED]&user_text=[REDACTED]',
|
||||
'url.path': '/custom-action',
|
||||
'url.query': 'api_key=[REDACTED]&user_text=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('LC_ACTION_QUERY_SECRET_67890');
|
||||
expect(JSON.stringify(attributes)).not.toContain('sensitive+prompt+words');
|
||||
});
|
||||
|
||||
it('redacts delimiter-less outgoing query segments and preserves separate HTTP ports', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startOutgoingSpanHook = instrumentationOptions?.startOutgoingSpanHook;
|
||||
|
||||
if (!startOutgoingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startOutgoingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startOutgoingSpanHook({
|
||||
protocol: 'https:',
|
||||
host: 'api.example.com',
|
||||
port: 8443,
|
||||
path: '/custom-action?LC_ACTION_QUERY_SECRET_67890&user_text=sensitive+prompt+words',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/custom-action?[REDACTED]&user_text=[REDACTED]',
|
||||
'http.url': 'https://api.example.com:8443/custom-action?[REDACTED]&user_text=[REDACTED]',
|
||||
'url.full': 'https://api.example.com:8443/custom-action?[REDACTED]&user_text=[REDACTED]',
|
||||
'url.path': '/custom-action',
|
||||
'url.query': '[REDACTED]&user_text=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('LC_ACTION_QUERY_SECRET_67890');
|
||||
expect(JSON.stringify(attributes)).not.toContain('sensitive+prompt+words');
|
||||
});
|
||||
|
||||
it('keeps outgoing HTTP URLs absolute when request options omit an origin', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startOutgoingSpanHook = instrumentationOptions?.startOutgoingSpanHook;
|
||||
|
||||
if (!startOutgoingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startOutgoingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startOutgoingSpanHook({
|
||||
protocol: 'http:',
|
||||
path: '/custom-action?api_key=LC_ACTION_QUERY_SECRET_67890',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/custom-action?api_key=[REDACTED]',
|
||||
'http.url': 'http://localhost/custom-action?api_key=[REDACTED]',
|
||||
'url.full': 'http://localhost/custom-action?api_key=[REDACTED]',
|
||||
'url.path': '/custom-action',
|
||||
'url.query': 'api_key=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('LC_ACTION_QUERY_SECRET_67890');
|
||||
});
|
||||
|
||||
it('uses the agent protocol for outgoing URL attributes when request protocol is absent', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startOutgoingSpanHook = instrumentationOptions?.startOutgoingSpanHook;
|
||||
|
||||
if (!startOutgoingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startOutgoingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startOutgoingSpanHook({
|
||||
agent: new HttpsAgent(),
|
||||
hostname: 'api.example.com',
|
||||
path: '/custom-action?api_key=LC_ACTION_QUERY_SECRET_67890',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/custom-action?api_key=[REDACTED]',
|
||||
'http.url': 'https://api.example.com/custom-action?api_key=[REDACTED]',
|
||||
'url.full': 'https://api.example.com/custom-action?api_key=[REDACTED]',
|
||||
'url.path': '/custom-action',
|
||||
'url.query': 'api_key=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('LC_ACTION_QUERY_SECRET_67890');
|
||||
});
|
||||
|
||||
it('does not infer HTTPS from port 443 without protocol context', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startOutgoingSpanHook = instrumentationOptions?.startOutgoingSpanHook;
|
||||
|
||||
if (!startOutgoingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startOutgoingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startOutgoingSpanHook({
|
||||
hostname: 'api.example.com',
|
||||
port: 443,
|
||||
path: '/custom-action?api_key=LC_ACTION_QUERY_SECRET_67890',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/custom-action?api_key=[REDACTED]',
|
||||
'http.url': 'http://api.example.com:443/custom-action?api_key=[REDACTED]',
|
||||
'url.full': 'http://api.example.com:443/custom-action?api_key=[REDACTED]',
|
||||
'url.path': '/custom-action',
|
||||
'url.query': 'api_key=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('LC_ACTION_QUERY_SECRET_67890');
|
||||
});
|
||||
|
||||
it('brackets IPv6 hostnames in outgoing HTTP URL attributes', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
|
||||
const startOutgoingSpanHook = instrumentationOptions?.startOutgoingSpanHook;
|
||||
|
||||
if (!startOutgoingSpanHook) {
|
||||
throw new Error('HTTP instrumentation startOutgoingSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startOutgoingSpanHook({
|
||||
protocol: 'http:',
|
||||
hostname: '::1',
|
||||
port: 8080,
|
||||
path: '/custom-action?api_key=LC_ACTION_QUERY_SECRET_67890',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/custom-action?api_key=[REDACTED]',
|
||||
'http.url': 'http://[::1]:8080/custom-action?api_key=[REDACTED]',
|
||||
'url.full': 'http://[::1]:8080/custom-action?api_key=[REDACTED]',
|
||||
'url.path': '/custom-action',
|
||||
'url.query': 'api_key=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('LC_ACTION_QUERY_SECRET_67890');
|
||||
});
|
||||
|
||||
it('redacts outgoing Undici URL query attributes before fetch spans are exported', () => {
|
||||
initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
const instrumentationOptions = mockUndiciInstrumentation.mock.calls[0]?.[0];
|
||||
const startSpanHook = instrumentationOptions?.startSpanHook;
|
||||
|
||||
if (!startSpanHook) {
|
||||
throw new Error('Undici instrumentation startSpanHook was not configured');
|
||||
}
|
||||
|
||||
const attributes = startSpanHook({
|
||||
origin: 'https://api.openweathermap.org',
|
||||
path: '/data/3.0/onecall?appid=OPENWEATHER_SECRET_123&lat=40.71&lon=-74.01',
|
||||
});
|
||||
|
||||
expect(attributes).toEqual({
|
||||
'http.target': '/data/3.0/onecall?appid=[REDACTED]&lat=[REDACTED]&lon=[REDACTED]',
|
||||
'http.url':
|
||||
'https://api.openweathermap.org/data/3.0/onecall?appid=[REDACTED]&lat=[REDACTED]&lon=[REDACTED]',
|
||||
'url.full':
|
||||
'https://api.openweathermap.org/data/3.0/onecall?appid=[REDACTED]&lat=[REDACTED]&lon=[REDACTED]',
|
||||
'url.path': '/data/3.0/onecall',
|
||||
'url.query': 'appid=[REDACTED]&lat=[REDACTED]&lon=[REDACTED]',
|
||||
});
|
||||
expect(JSON.stringify(attributes)).not.toContain('OPENWEATHER_SECRET_123');
|
||||
expect(JSON.stringify(attributes)).not.toContain('40.71');
|
||||
expect(JSON.stringify(attributes)).not.toContain('-74.01');
|
||||
});
|
||||
|
||||
it('reflects lifecycle status from the controller getter', async () => {
|
||||
const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { MongooseInstrumentation } from '@opentelemetry/instrumentation-mongoose
|
|||
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
||||
import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node';
|
||||
import type { Span, Attributes } from '@opentelemetry/api';
|
||||
import type { RequestOptions } from 'node:http';
|
||||
import type { TelemetryConfig, TelemetryStatus } from './config';
|
||||
import { getTelemetryConfig } from './config';
|
||||
|
||||
|
|
@ -28,6 +29,21 @@ interface RegisteredSignal {
|
|||
listener: NodeJS.SignalsListener;
|
||||
}
|
||||
|
||||
interface RequestUrlParts {
|
||||
href?: string;
|
||||
search?: string;
|
||||
pathname?: string;
|
||||
}
|
||||
|
||||
interface AgentProtocol {
|
||||
protocol?: string;
|
||||
}
|
||||
|
||||
interface UndiciRequestInfo {
|
||||
path?: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
let activeSdk: NodeSDK | undefined;
|
||||
let pendingSdk: NodeSDK | undefined;
|
||||
let startPromise: Promise<void> | undefined;
|
||||
|
|
@ -95,6 +111,183 @@ function getSanitizedIncomingUrlAttributes(
|
|||
return attributes;
|
||||
}
|
||||
|
||||
function getStringValue(value: string | number | null | undefined): string | undefined {
|
||||
if (value == null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const stringValue = String(value).trim();
|
||||
return stringValue || undefined;
|
||||
}
|
||||
|
||||
function getRedactedQuery(search: string): string | undefined {
|
||||
const query = search.startsWith('?') ? search.slice(1) : search;
|
||||
if (!query) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return query
|
||||
.split('&')
|
||||
.map((part) => {
|
||||
const separatorIndex = part.indexOf('=');
|
||||
if (separatorIndex < 0) {
|
||||
return REDACTED_QUERY_VALUE;
|
||||
}
|
||||
|
||||
const key = part.slice(0, separatorIndex);
|
||||
if (!key) {
|
||||
return REDACTED_QUERY_VALUE;
|
||||
}
|
||||
|
||||
return `${key}=${REDACTED_QUERY_VALUE}`;
|
||||
})
|
||||
.join('&');
|
||||
}
|
||||
|
||||
function getSanitizedUrlAttributesFromParts(
|
||||
origin: string | undefined,
|
||||
pathname: string,
|
||||
search: string,
|
||||
): Attributes {
|
||||
const path = pathname || '/';
|
||||
const redactedQuery = getRedactedQuery(search);
|
||||
const target = redactedQuery ? `${path}?${redactedQuery}` : path;
|
||||
const fullUrl = origin ? `${origin}${target}` : target;
|
||||
const attributes: Attributes = {
|
||||
'http.target': target,
|
||||
'http.url': fullUrl,
|
||||
'url.full': fullUrl,
|
||||
'url.path': path,
|
||||
};
|
||||
|
||||
if (redactedQuery) {
|
||||
attributes['url.query'] = redactedQuery;
|
||||
}
|
||||
|
||||
return attributes;
|
||||
}
|
||||
|
||||
function getFallbackUrlParts(rawUrl: string): { pathname: string; search: string } {
|
||||
const queryIndex = rawUrl.indexOf('?');
|
||||
if (queryIndex < 0) {
|
||||
return { pathname: rawUrl || '/', search: '' };
|
||||
}
|
||||
|
||||
return {
|
||||
pathname: rawUrl.slice(0, queryIndex) || '/',
|
||||
search: rawUrl.slice(queryIndex),
|
||||
};
|
||||
}
|
||||
|
||||
function getSanitizedOutgoingUrlAttributes(rawUrl: string, origin?: string): Attributes {
|
||||
const hasOrigin = /^[a-z][a-z\d+\-.]*:\/\//i.test(rawUrl);
|
||||
|
||||
try {
|
||||
const parsedUrl = new URL(rawUrl, origin ?? 'http://localhost');
|
||||
const safeOrigin = hasOrigin || origin ? parsedUrl.origin : undefined;
|
||||
return getSanitizedUrlAttributesFromParts(safeOrigin, parsedUrl.pathname, parsedUrl.search);
|
||||
} catch {
|
||||
const { pathname, search } = getFallbackUrlParts(rawUrl);
|
||||
return getSanitizedUrlAttributesFromParts(origin, pathname, search);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeProtocol(protocol: string): string {
|
||||
return protocol.endsWith(':') ? protocol : `${protocol}:`;
|
||||
}
|
||||
|
||||
function getRequestAgentProtocol(request: RequestOptions): string | undefined {
|
||||
const { agent } = request;
|
||||
if (!agent || typeof agent === 'boolean') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return getStringValue((agent as AgentProtocol).protocol);
|
||||
}
|
||||
|
||||
function getOutgoingHttpProtocol(request: RequestOptions): string {
|
||||
const protocol = getStringValue(request.protocol) ?? getRequestAgentProtocol(request);
|
||||
if (!protocol) {
|
||||
return 'http:';
|
||||
}
|
||||
|
||||
return normalizeProtocol(protocol);
|
||||
}
|
||||
|
||||
function getUrlAuthorityHost(host: string): string {
|
||||
try {
|
||||
const parsedHost = new URL(`http://${host}`);
|
||||
return parsedHost.host;
|
||||
} catch {
|
||||
if (host.includes(':') && !host.startsWith('[')) {
|
||||
return `[${host}]`;
|
||||
}
|
||||
}
|
||||
|
||||
return host;
|
||||
}
|
||||
|
||||
function getHostWithPort(host: string, port: string | undefined): string {
|
||||
const authorityHost = getUrlAuthorityHost(host);
|
||||
if (!port) {
|
||||
return authorityHost;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedHost = new URL(`http://${authorityHost}`);
|
||||
if (parsedHost.port) {
|
||||
return authorityHost;
|
||||
}
|
||||
} catch {
|
||||
return authorityHost;
|
||||
}
|
||||
|
||||
return `${authorityHost}:${port}`;
|
||||
}
|
||||
|
||||
function getOutgoingHttpOrigin(request: RequestOptions): string | undefined {
|
||||
const protocol = getOutgoingHttpProtocol(request);
|
||||
const host = getStringValue(request.host);
|
||||
const port = getStringValue(request.port);
|
||||
if (host) {
|
||||
return `${protocol}//${getHostWithPort(host, port)}`;
|
||||
}
|
||||
|
||||
const hostname = getStringValue(request.hostname);
|
||||
const resolvedHostname = hostname ?? 'localhost';
|
||||
return `${protocol}//${getHostWithPort(resolvedHostname, port)}`;
|
||||
}
|
||||
|
||||
function getOutgoingHttpUrl(request: RequestOptions & RequestUrlParts): string {
|
||||
if (request.path) {
|
||||
return request.path;
|
||||
}
|
||||
|
||||
if (request.href) {
|
||||
return request.href;
|
||||
}
|
||||
|
||||
const pathname = request.pathname || '/';
|
||||
if (!request.search) {
|
||||
return pathname;
|
||||
}
|
||||
|
||||
const search = request.search.startsWith('?') ? request.search : `?${request.search}`;
|
||||
return `${pathname}${search}`;
|
||||
}
|
||||
|
||||
function getSanitizedOutgoingHttpUrlAttributes(request: RequestOptions): Attributes {
|
||||
const requestWithUrlParts = request as RequestOptions & RequestUrlParts;
|
||||
return getSanitizedOutgoingUrlAttributes(
|
||||
getOutgoingHttpUrl(requestWithUrlParts),
|
||||
getOutgoingHttpOrigin(request),
|
||||
);
|
||||
}
|
||||
|
||||
function getSanitizedUndiciUrlAttributes(request: UndiciRequestInfo): Attributes {
|
||||
return getSanitizedOutgoingUrlAttributes(request.path ?? '/', request.origin);
|
||||
}
|
||||
|
||||
function getResourceAttributes(config: TelemetryConfig): Attributes {
|
||||
const attributes: Attributes = {
|
||||
[ATTR_SERVICE_NAME]: config.serviceName,
|
||||
|
|
@ -123,6 +316,8 @@ function createSdk(config: TelemetryConfig): NodeSDK {
|
|||
},
|
||||
startIncomingSpanHook: (request: IncomingMessage) =>
|
||||
getSanitizedIncomingUrlAttributes(request, config.healthPath),
|
||||
startOutgoingSpanHook: (request: RequestOptions) =>
|
||||
getSanitizedOutgoingHttpUrlAttributes(request),
|
||||
ignoreIncomingRequestHook: (request: IncomingMessage) =>
|
||||
shouldIgnoreIncomingRequest(request, config.healthPath),
|
||||
}),
|
||||
|
|
@ -130,7 +325,9 @@ function createSdk(config: TelemetryConfig): NodeSDK {
|
|||
new MongoDBInstrumentation(),
|
||||
new MongooseInstrumentation(),
|
||||
new IORedisInstrumentation(),
|
||||
new UndiciInstrumentation(),
|
||||
new UndiciInstrumentation({
|
||||
startSpanHook: (request: UndiciRequestInfo) => getSanitizedUndiciUrlAttributes(request),
|
||||
}),
|
||||
],
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue