🪭 feat: Add opt-in Langfuse fanout gateway + collector (#13872)

* feat: add opt-in Langfuse fanout collector * feat: fan out Langfuse feedback scores * docs: prepare Langfuse fanout for OSS setup * fix: clarify Langfuse fanout collector config * test: stabilize librechat suite * test: fix upload dialog import order * fix: omit empty Langfuse tenant fields * fix: gate tenant Langfuse fanout * test: cover central Langfuse env fallback * style: format Langfuse fanout config * feat: route langfuse fanout by destination * docs: clarify langfuse compose destination scope * test: remove unrelated suite stabilization * style: sort agent imports * fix: treat blank tenant fanout toggle as disabled * fix: rename tenant fanout emergency toggle * test: guard langfuse fanout collector config drift * feat: tune langfuse fanout batching * test: render fanout helm tests without dependencies * fix: narrow remote agent run config * refactor: share string normalization helper * fix: align langfuse fanout env parsing * fix(langfuse): align score fanout toggles with traces * fix(langfuse): keep central fanout config collector-only * fix(langfuse): type fanout collector config * fix(langfuse): harden tenant fanout config * feat(langfuse): support media fanout gateway * fix(langfuse): route tenant fanout through destination URL * fix(langfuse): harden fanout routing checks * ci(langfuse): test fanout gateway changes * ci(langfuse): check fanout go formatting * fix(langfuse): satisfy api typecheck
2026-10-20 17:13:44 +00:00 · 2026-06-26 17:26:39 +02:00 · 2026-06-26 17:26:39 +02:00 · a0529c9af7
commit a0529c9af7
parent 0789a04d11
38 changed files with 5699 additions and 130 deletions
--- a/helm/librechat/readme.md
+++ b/helm/librechat/readme.md
@ -53,3 +53,45 @@ also register this LibreChat callback URL with your identity provider:
 ```text
 https://<librechat-domain>/api/admin/oauth/openid/callback
 ```
+
+## Langfuse Fanout
+
+The chart can optionally deploy a Langfuse fanout gateway with an internal
+OpenTelemetry Collector sidecar. The gateway handles Langfuse media fanout and
+proxies traces to the collector; the collector forwards tenant-scoped Langfuse
+traces to both a central Langfuse project and the tenant Langfuse project. It is
+disabled by default.
+
+When enabled, the chart also sets `LANGFUSE_FANOUT_ENABLED` and
+`LANGFUSE_FANOUT_COLLECTOR_URL` for the LibreChat app unless those values are
+already provided in `librechat.configEnv`.
+
+Set `librechat.configEnv.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=true` to keep
+central trace export flowing through the fanout gateway while disabling tenant trace
+and score export. When omitted, false, or blank, tenant export remains available
+if tenant keys and a known destination are configured.
+
+Langfuse tenant base URLs are selected from the startup-configured destination
+map rendered into LibreChat and the fanout gateway. Tenant API keys can still be added
+through tenant app configuration at runtime without restarting either component.
+The internal collector provides trace memory limiting, batching, tenant routing,
+and removal of LibreChat-only routing attributes before export.
+
+The fanout gateway stores one-time media upload plans in Redis so media create
+and byte-upload requests can land on different gateway replicas. Set
+`langfuseFanout.redis.uri` for an external Redis service, or enable the bundled
+Redis chart with `redis.enabled=true` and let the chart derive the internal URI.
+Scale the gateway manually with `langfuseFanout.replicaCount`; the chart does
+not create a fanout HPA.
+The internal collector receiver is bound to `127.0.0.1:4319` by default because
+only the gateway sidecar should send traces to it.
+
+The gateway exposes Prometheus metrics at `/metrics`. Configure
+`langfuseFanout.metrics.secret.name` and `.key` to pass a bearer token secret to
+the gateway; if omitted, `/metrics` returns 401. Use
+`langfuseFanout.service.annotations` for scrape annotations when your cluster
+uses annotation-based discovery. The gateway container also has configurable
+`/healthz` liveness and readiness probes under `langfuseFanout`.
+
+See [`otel/langfuse-fanout/README.md`](../../otel/langfuse-fanout/README.md)
+for the central Langfuse secret and values example.
--- a/helm/librechat/templates/_helpers.tpl
+++ b/helm/librechat/templates/_helpers.tpl
@ -28,13 +28,21 @@ If release name contains chart name it will be used as a full name.
 Common labels
 */}}
 {{- define "librechat.labels" -}}
-helm.sh/chart: {{ include "librechat.chart" . }}
-{{ include "librechat.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- include "librechat.standardLabels" (dict "root" . "selectorLabels" (include "librechat.selectorLabels" .)) }}
 {{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
+
+{{/*
+Standard labels for chart-managed workloads.
+*/}}
+{{- define "librechat.standardLabels" -}}
+{{- $root := .root -}}
+helm.sh/chart: {{ include "librechat.chart" $root }}
+{{ .selectorLabels }}
+{{- if $root.Chart.AppVersion }}
+app.kubernetes.io/version: {{ $root.Chart.AppVersion | quote }}
 {{- end }}
+app.kubernetes.io/managed-by: {{ $root.Release.Service }}
+{{- end -}}

 {{/*
 Selector labels
@ -44,6 +52,96 @@ app.kubernetes.io/name: {{ include "librechat.fullname" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}

+{{/*
+Langfuse fanout collector service name.
+*/}}
+{{- define "librechat.langfuseFanout.fullname" -}}
+{{- printf "%s-langfuse-fanout" (include "librechat.fullname" .) | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Langfuse fanout collector selector labels.
+*/}}
+{{- define "librechat.langfuseFanout.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "librechat.langfuseFanout.fullname" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Langfuse fanout collector labels.
+*/}}
+{{- define "librechat.langfuseFanout.labels" -}}
+{{- include "librechat.standardLabels" (dict "root" . "selectorLabels" (include "librechat.langfuseFanout.selectorLabels" .)) }}
+{{- end }}
+
+{{/*
+Validate Langfuse fanout destination keys. LibreChat normalizes destination
+keys to lowercase before putting them on trace attributes, so Helm values must
+already use the same lowercase key shape for collector routing to match.
+*/}}
+{{- define "librechat.langfuseFanout.validateDestinationKey" -}}
+{{- $name := printf "%v" . -}}
+{{- if not (regexMatch "^[a-z][a-z0-9_-]*$" $name) -}}
+{{- fail (printf "langfuseFanout.tenant.destinations key %q is invalid; use lowercase keys matching ^[a-z][a-z0-9_-]*$ so LibreChat trace attributes match collector routes" $name) -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Render the environment variable name used by the collector for a destination.
+*/}}
+{{- define "librechat.langfuseFanout.destinationBaseUrlEnvName" -}}
+{{- printf "LANGFUSE_FANOUT_TENANT_%s_BASE_URL" (. | printf "%v" | upper | replace "-" "_") -}}
+{{- end }}
+
+{{/*
+Validate the full destination key set. Destination keys can contain hyphens and
+underscores, but the collector base URL env vars replace hyphens with
+underscores. Reject keys such as foo-bar and foo_bar because they would render
+the same LANGFUSE_FANOUT_TENANT_FOO_BAR_BASE_URL env var.
+*/}}
+{{- define "librechat.langfuseFanout.validateDestinationKeys" -}}
+{{- $seenEnvNames := dict -}}
+{{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations -}}
+{{- include "librechat.langfuseFanout.validateDestinationKey" $name -}}
+{{- $envName := include "librechat.langfuseFanout.destinationBaseUrlEnvName" $name -}}
+{{- if hasKey $seenEnvNames $envName -}}
+{{- fail (printf "langfuseFanout.tenant.destinations keys %q and %q both render %s; use destination keys that remain unique after uppercasing and replacing '-' with '_' for env vars" (get $seenEnvNames $envName) $name $envName) -}}
+{{- end -}}
+{{- $_ := set $seenEnvNames $envName $name -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Render the fanout destination list consumed by LibreChat and the fanout gateway.
+*/}}
+{{- define "librechat.langfuseFanout.tenantDestinationsEnv" -}}
+{{- include "librechat.langfuseFanout.validateDestinationKeys" . -}}
+{{- $tenantDestinations := list -}}
+{{- range $name, $destination := .Values.langfuseFanout.tenant.destinations -}}
+{{- $tenantDestinations = append $tenantDestinations (printf "%s=%s" $name $destination.baseUrl) -}}
+{{- end -}}
+{{- join "," $tenantDestinations -}}
+{{- end }}
+
+{{/*
+Render the fanout destination key list consumed by the gateway as a startup
+guard against media destinations the collector cannot route traces to.
+*/}}
+{{- define "librechat.langfuseFanout.tenantDestinationKeysEnv" -}}
+{{- include "librechat.langfuseFanout.validateDestinationKeys" . -}}
+{{- $tenantDestinationKeys := list -}}
+{{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations -}}
+{{- $tenantDestinationKeys = append $tenantDestinationKeys $name -}}
+{{- end -}}
+{{- join "," $tenantDestinationKeys -}}
+{{- end }}
+
+{{/*
+Bundled Redis URI used when the Redis subchart is enabled.
+*/}}
+{{- define "librechat.bundledRedisURI" -}}
+{{- printf "redis://%s-master.%s.svc.cluster.local:6379" (include "common.names.fullname" .Subcharts.redis) (.Release.Namespace | lower) -}}
+{{- end }}

 {{/*
 RAG Selector labels
--- a/helm/librechat/templates/configmap-env.yaml
+++ b/helm/librechat/templates/configmap-env.yaml
@ -19,11 +19,25 @@ data:
  USE_REDIS: "true"
  {{- end }}
  {{- if and (not (dig "configEnv" "REDIS_URI" "" .Values.librechat)) .Values.redis.enabled }}
-  REDIS_URI: redis://{{ include "common.names.fullname" .Subcharts.redis }}-master.{{ .Release.Namespace | lower }}.svc.cluster.local:6379
+  REDIS_URI: {{ include "librechat.bundledRedisURI" . }}
  {{- end }}
  {{- if and $adminPanelUrl (not $configAdminPanelUrl) }}
  ADMIN_PANEL_URL: {{ $adminPanelUrl | quote }}
  {{- end }}
+  {{- if and .Values.langfuseFanout.enabled (not (hasKey $configEnv "LANGFUSE_FANOUT_ENABLED")) }}
+  LANGFUSE_FANOUT_ENABLED: "true"
+  {{- end }}
+  {{- if and .Values.langfuseFanout.enabled (not (hasKey $configEnv "LANGFUSE_FANOUT_COLLECTOR_URL")) }}
+  LANGFUSE_FANOUT_COLLECTOR_URL: http://{{ include "librechat.langfuseFanout.fullname" . }}.{{ .Release.Namespace | lower }}.svc.cluster.local:{{ .Values.langfuseFanout.service.port }}
+  {{- end }}
+  {{- if and .Values.langfuseFanout.enabled (not (hasKey $configEnv "LANGFUSE_FANOUT_TENANT_DESTINATIONS")) }}
+  {{- include "librechat.langfuseFanout.validateDestinationKeys" . }}
+  {{- $tenantDestinations := list }}
+  {{- range $name, $destination := .Values.langfuseFanout.tenant.destinations }}
+  {{- $tenantDestinations = append $tenantDestinations (printf "%s=%s" $name $destination.baseUrl) }}
+  {{- end }}
+  LANGFUSE_FANOUT_TENANT_DESTINATIONS: {{ join "," $tenantDestinations | quote }}
+  {{- end }}
  {{- if $configEnv }}
  {{- $renderedConfigEnv := $configEnv }}
  {{- if and $adminPanelUrl (hasKey $configEnv "ADMIN_PANEL_URL") (not $configAdminPanelUrl) }}
--- a/helm/librechat/templates/langfuse-fanout-configmap.yaml
+++ b/helm/librechat/templates/langfuse-fanout-configmap.yaml
@ -0,0 +1,101 @@
+{{- if .Values.langfuseFanout.enabled }}
+{{- include "librechat.langfuseFanout.validateDestinationKeys" . }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "librechat.langfuseFanout.fullname" . }}-config
+  labels:
+    {{- include "librechat.langfuseFanout.labels" . | nindent 4 }}
+data:
+  otelcol.yaml: |
+    extensions:
+      headers_setter/tenant_passthrough:
+        headers:
+          - action: upsert
+            key: Authorization
+            from_context: authorization
+
+    receivers:
+      otlp:
+        protocols:
+          http:
+            endpoint: ${env:LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT}
+            include_metadata: true
+            traces_url_path: /api/public/otel/v1/traces
+
+    connectors:
+      routing/langfuse_tenant_destination:
+        error_mode: ignore
+        table:
+          {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}
+          {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}
+          - context: span
+            condition: attributes["librechat.langfuse.destination"] == {{ $name | quote }}
+            pipelines: [traces/tenant_{{ $name }}]
+          {{- end }}
+
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_mib: ${env:LANGFUSE_FANOUT_MEMORY_LIMIT_MIB}
+        spike_limit_mib: ${env:LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB}
+      filter/tenant_export:
+        error_mode: ignore
+        traces:
+          span:
+            - attributes["librechat.langfuse.tenant_export.enabled"] != "true"
+      attributes/drop_librechat_routing:
+        actions:
+          - key: librechat.langfuse.tenant_export.enabled
+            action: delete
+          - key: librechat.langfuse.destination
+            action: delete
+      batch/central:
+        timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}
+        send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}
+      {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}
+      {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}
+      batch/by_auth_{{ $name }}:
+        timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}
+        send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}
+        metadata_keys: [authorization]
+        metadata_cardinality_limit: ${env:LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT}
+      {{- end }}
+
+    exporters:
+      otlphttp/central:
+        # Langfuse Cloud base URL options: https://cloud.langfuse.com (EU),
+        # https://us.cloud.langfuse.com (US), https://jp.cloud.langfuse.com (JP).
+        endpoint: "${env:LANGFUSE_FANOUT_CENTRAL_BASE_URL}/api/public/otel"
+        headers:
+          Authorization: "${env:LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER}"
+          x-langfuse-ingestion-version: "4"
+      {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}
+      {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}
+      otlphttp/tenant_{{ $name }}:
+        endpoint: "${env:{{ include "librechat.langfuseFanout.destinationBaseUrlEnvName" $name }}}/api/public/otel"
+        auth:
+          authenticator: headers_setter/tenant_passthrough
+        headers:
+          x-langfuse-ingestion-version: "4"
+      {{- end }}
+
+    service:
+      extensions: [headers_setter/tenant_passthrough]
+      pipelines:
+        traces/central:
+          receivers: [otlp]
+          processors: [memory_limiter, attributes/drop_librechat_routing, batch/central]
+          exporters: [otlphttp/central]
+        traces/tenant:
+          receivers: [otlp]
+          processors: [memory_limiter, filter/tenant_export]
+          exporters: [routing/langfuse_tenant_destination]
+        {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}
+        {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}
+        traces/tenant_{{ $name }}:
+          receivers: [routing/langfuse_tenant_destination]
+          processors: [attributes/drop_librechat_routing, batch/by_auth_{{ $name }}]
+          exporters: [otlphttp/tenant_{{ $name }}]
+        {{- end }}
+{{- end }}
--- a/helm/librechat/templates/langfuse-fanout-deployment.yaml
+++ b/helm/librechat/templates/langfuse-fanout-deployment.yaml
@ -0,0 +1,135 @@
+{{- if .Values.langfuseFanout.enabled }}
+{{- include "librechat.langfuseFanout.validateDestinationKeys" . }}
+{{- $redisURI := .Values.langfuseFanout.redis.uri }}
+{{- if and (not $redisURI) .Values.redis.enabled }}
+{{- $redisURI = include "librechat.bundledRedisURI" . }}
+{{- end }}
+{{- if not $redisURI }}
+{{- fail "langfuseFanout.redis.uri is required when langfuseFanout.enabled=true unless redis.enabled=true" }}
+{{- end }}
+{{- $publicURL := .Values.langfuseFanout.publicUrl }}
+{{- if not $publicURL }}
+{{- $publicURL = printf "http://%s.%s.svc.cluster.local:%v" (include "librechat.langfuseFanout.fullname" .) (.Release.Namespace | lower) .Values.langfuseFanout.service.port }}
+{{- end }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "librechat.langfuseFanout.fullname" . }}
+  labels:
+    {{- include "librechat.langfuseFanout.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.langfuseFanout.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "librechat.langfuseFanout.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        checksum/langfuse-fanout-config: {{ toYaml .Values.langfuseFanout | sha256sum }}
+        {{- with .Values.langfuseFanout.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "librechat.langfuseFanout.labels" . | nindent 8 }}
+        {{- with .Values.langfuseFanout.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      containers:
+        - name: langfuse-fanout
+          image: "{{ .Values.langfuseFanout.image.repository }}:{{ .Values.langfuseFanout.image.tag }}"
+          imagePullPolicy: {{ .Values.langfuseFanout.image.pullPolicy }}
+          ports:
+            - name: otlp-http
+              containerPort: 4318
+              protocol: TCP
+          env:
+            - name: LANGFUSE_FANOUT_CENTRAL_BASE_URL
+              value: {{ .Values.langfuseFanout.central.baseUrl | quote }}
+            - name: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER
+              valueFrom:
+                secretKeyRef:
+                  name: {{ required "langfuseFanout.central.authHeaderSecret.name is required when langfuseFanout.enabled=true" .Values.langfuseFanout.central.authHeaderSecret.name | quote }}
+                  key: {{ .Values.langfuseFanout.central.authHeaderSecret.key | quote }}
+            - name: LANGFUSE_FANOUT_TENANT_DESTINATIONS
+              value: {{ include "librechat.langfuseFanout.tenantDestinationsEnv" . | quote }}
+            - name: LANGFUSE_FANOUT_UPSTREAM_TIMEOUT
+              value: {{ .Values.langfuseFanout.upstreamTimeout | quote }}
+            - name: LANGFUSE_FANOUT_TRACE_COLLECTOR_URL
+              value: {{ .Values.langfuseFanout.traceCollectorUrl | quote }}
+            - name: LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS
+              value: {{ include "librechat.langfuseFanout.tenantDestinationKeysEnv" . | quote }}
+            - name: LANGFUSE_FANOUT_REDIS_URI
+              value: {{ $redisURI | quote }}
+            {{- with .Values.langfuseFanout.redis.username }}
+            - name: LANGFUSE_FANOUT_REDIS_USERNAME
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .Values.langfuseFanout.redis.passwordSecret.name }}
+            - name: LANGFUSE_FANOUT_REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ . | quote }}
+                  key: {{ $.Values.langfuseFanout.redis.passwordSecret.key | quote }}
+            {{- end }}
+            - name: LANGFUSE_FANOUT_REDIS_KEY_PREFIX
+              value: {{ .Values.langfuseFanout.redis.keyPrefix | quote }}
+            - name: LANGFUSE_FANOUT_PUBLIC_URL
+              value: {{ $publicURL | quote }}
+            {{- with .Values.langfuseFanout.metrics.secret.name }}
+            - name: LANGFUSE_FANOUT_METRICS_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ . | quote }}
+                  key: {{ $.Values.langfuseFanout.metrics.secret.key | quote }}
+            {{- end }}
+          livenessProbe:
+            {{- toYaml .Values.langfuseFanout.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml .Values.langfuseFanout.readinessProbe | nindent 12 }}
+          resources:
+            {{- toYaml .Values.langfuseFanout.resources | nindent 12 }}
+        - name: otelcol
+          image: "{{ .Values.langfuseFanout.otelCollector.image.repository }}:{{ .Values.langfuseFanout.otelCollector.image.tag }}"
+          imagePullPolicy: {{ .Values.langfuseFanout.otelCollector.image.pullPolicy }}
+          args: ["--config=/etc/otelcol/otelcol.yaml"]
+          ports:
+            - name: otlp-internal
+              containerPort: 4319
+              protocol: TCP
+          env:
+            - name: LANGFUSE_FANOUT_CENTRAL_BASE_URL
+              value: {{ .Values.langfuseFanout.central.baseUrl | quote }}
+            - name: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER
+              valueFrom:
+                secretKeyRef:
+                  name: {{ required "langfuseFanout.central.authHeaderSecret.name is required when langfuseFanout.enabled=true" .Values.langfuseFanout.central.authHeaderSecret.name | quote }}
+                  key: {{ .Values.langfuseFanout.central.authHeaderSecret.key | quote }}
+            - name: LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT
+              value: {{ .Values.langfuseFanout.otelCollector.receiverEndpoint | quote }}
+            {{- range $name, $destination := .Values.langfuseFanout.tenant.destinations }}
+            - name: {{ include "librechat.langfuseFanout.destinationBaseUrlEnvName" $name }}
+              value: {{ $destination.baseUrl | quote }}
+            {{- end }}
+            - name: LANGFUSE_FANOUT_MEMORY_LIMIT_MIB
+              value: {{ .Values.langfuseFanout.memoryLimitMiB | quote }}
+            - name: LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB
+              value: {{ .Values.langfuseFanout.memorySpikeLimitMiB | quote }}
+            - name: LANGFUSE_FANOUT_BATCH_TIMEOUT
+              value: {{ .Values.langfuseFanout.batchTimeout | quote }}
+            - name: LANGFUSE_FANOUT_BATCH_SEND_SIZE
+              value: {{ .Values.langfuseFanout.batchSendSize | quote }}
+            - name: LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT
+              value: {{ .Values.langfuseFanout.metadataCardinalityLimit | quote }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/otelcol/otelcol.yaml
+              subPath: otelcol.yaml
+              readOnly: true
+          resources:
+            {{- toYaml .Values.langfuseFanout.otelCollector.resources | nindent 12 }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "librechat.langfuseFanout.fullname" . }}-config
+{{- end }}
--- a/helm/librechat/templates/langfuse-fanout-service.yaml
+++ b/helm/librechat/templates/langfuse-fanout-service.yaml
@ -0,0 +1,23 @@
+{{- if .Values.langfuseFanout.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "librechat.langfuseFanout.fullname" . }}
+  labels:
+    {{- include "librechat.langfuseFanout.labels" . | nindent 4 }}
+  {{- with .Values.langfuseFanout.service.annotations }}
+  annotations:
+    {{- range $key, $value := . }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+  {{- end }}
+spec:
+  type: {{ .Values.langfuseFanout.service.type }}
+  ports:
+    - name: otlp-http
+      port: {{ .Values.langfuseFanout.service.port }}
+      targetPort: otlp-http
+      protocol: TCP
+  selector:
+    {{- include "librechat.langfuseFanout.selectorLabels" . | nindent 4 }}
+{{- end }}
--- a/helm/librechat/tests/langfuse_fanout_selector_test.sh
+++ b/helm/librechat/tests/langfuse_fanout_selector_test.sh
@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Regression test for Langfuse fanout Helm selectors.
+#
+# The fanout collector must not share the main LibreChat app selector labels.
+# Otherwise the main Service can route HTTP traffic to the OTEL collector pod.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+REPO_ROOT="$(cd "${CHART_DIR}/../.." && pwd)"
+RENDER_CHART_DIR="$(mktemp -d -t librechat-fanout-chart.XXXXXX)"
+RENDERED_FILE="$(mktemp -t librechat-fanout-render.XXXXXX)"
+INVALID_RENDER_ERROR="$(mktemp -t librechat-fanout-invalid-key.XXXXXX)"
+COLLISION_RENDER_ERROR="$(mktemp -t librechat-fanout-colliding-key.XXXXXX)"
+trap 'rm -rf "${RENDER_CHART_DIR}"; rm -f "${RENDERED_FILE}" "${INVALID_RENDER_ERROR}" "${COLLISION_RENDER_ERROR}"' EXIT
+
+if ! command -v helm >/dev/null 2>&1; then
+  echo "FAIL: helm not on PATH" >&2
+  exit 1
+fi
+
+mkdir -p "${RENDER_CHART_DIR}/templates"
+awk '/^dependencies:/{ exit } { print }' "${CHART_DIR}/Chart.yaml" > "${RENDER_CHART_DIR}/Chart.yaml"
+cp "${CHART_DIR}/values.yaml" "${RENDER_CHART_DIR}/values.yaml"
+cp "${CHART_DIR}/templates/_helpers.tpl" "${RENDER_CHART_DIR}/templates/_helpers.tpl"
+cp "${CHART_DIR}/templates/service.yaml" "${RENDER_CHART_DIR}/templates/service.yaml"
+cp "${CHART_DIR}/templates/langfuse-fanout-service.yaml" \
+  "${RENDER_CHART_DIR}/templates/langfuse-fanout-service.yaml"
+cp "${CHART_DIR}/templates/langfuse-fanout-deployment.yaml" \
+  "${RENDER_CHART_DIR}/templates/langfuse-fanout-deployment.yaml"
+
+helm template librechat "${RENDER_CHART_DIR}" \
+  --set langfuseFanout.enabled=true \
+  --set langfuseFanout.central.authHeaderSecret.name=langfuse-central \
+  --set langfuseFanout.redis.uri=redis://langfuse-fanout-redis:6379 \
+  --show-only templates/service.yaml \
+  --show-only templates/langfuse-fanout-service.yaml \
+  --show-only templates/langfuse-fanout-deployment.yaml \
+  > "${RENDERED_FILE}"
+
+if helm template librechat "${RENDER_CHART_DIR}" \
+  --set langfuseFanout.enabled=true \
+  --set langfuseFanout.central.authHeaderSecret.name=langfuse-central \
+  --set langfuseFanout.redis.uri=redis://langfuse-fanout-redis:6379 \
+  --set langfuseFanout.tenant.destinations.EU.baseUrl=https://cloud.langfuse.com \
+  --show-only templates/langfuse-fanout-deployment.yaml \
+  > /dev/null 2> "${INVALID_RENDER_ERROR}"; then
+  echo "FAIL: Helm accepted invalid uppercase Langfuse fanout destination key" >&2
+  exit 1
+fi
+
+if ! grep -q 'langfuseFanout.tenant.destinations key "EU" is invalid' "${INVALID_RENDER_ERROR}"; then
+  echo "FAIL: invalid destination key render did not explain the key contract" >&2
+  cat "${INVALID_RENDER_ERROR}" >&2
+  exit 1
+fi
+
+if helm template librechat "${RENDER_CHART_DIR}" \
+  --set langfuseFanout.enabled=true \
+  --set langfuseFanout.central.authHeaderSecret.name=langfuse-central \
+  --set langfuseFanout.redis.uri=redis://langfuse-fanout-redis:6379 \
+  --set langfuseFanout.tenant.destinations.foo-bar.baseUrl=https://foo-bar.example.com \
+  --set langfuseFanout.tenant.destinations.foo_bar.baseUrl=https://foo-bar.example.com \
+  --show-only templates/langfuse-fanout-deployment.yaml \
+  > /dev/null 2> "${COLLISION_RENDER_ERROR}"; then
+  echo "FAIL: Helm accepted colliding Langfuse fanout destination env var keys" >&2
+  exit 1
+fi
+
+if ! grep -q 'both render LANGFUSE_FANOUT_TENANT_FOO_BAR_BASE_URL' "${COLLISION_RENDER_ERROR}"; then
+  echo "FAIL: colliding destination key render did not explain the env var collision" >&2
+  cat "${COLLISION_RENDER_ERROR}" >&2
+  exit 1
+fi
+
+if ! command -v node >/dev/null 2>&1; then
+  echo "FAIL: node not on PATH" >&2
+  exit 1
+fi
+
+NODE_PATH="${REPO_ROOT}/node_modules${NODE_PATH:+:${NODE_PATH}}" \
+RENDERED_FILE="${RENDERED_FILE}" node <<'NODE'
+const fs = require('fs');
+const yaml = require('js-yaml');
+
+const docs = yaml
+  .loadAll(fs.readFileSync(process.env.RENDERED_FILE, 'utf8'))
+  .filter(Boolean);
+
+function fail(message) {
+  console.error(`FAIL: ${message}`);
+  process.exit(1);
+}
+
+function find(kind, name) {
+  const doc = docs.find((candidate) => candidate.kind === kind && candidate.metadata?.name === name);
+  if (!doc) {
+    fail(`missing ${kind}/${name}`);
+  }
+  return doc;
+}
+
+function isSubset(subset, labels) {
+  return Object.entries(subset ?? {}).every(([key, value]) => labels?.[key] === value);
+}
+
+function envValue(env, name) {
+  return (env ?? []).find((entry) => entry.name === name)?.value;
+}
+
+const mainService = find('Service', 'librechat-librechat');
+const fanoutService = find('Service', 'librechat-librechat-langfuse-fanout');
+const fanoutDeployment = find('Deployment', 'librechat-librechat-langfuse-fanout');
+const fanoutContainer = fanoutDeployment.spec?.template?.spec?.containers?.find(
+  (container) => container.name === 'langfuse-fanout',
+);
+if (!fanoutContainer) {
+  fail('missing langfuse-fanout container');
+}
+
+const mainSelector = mainService.spec?.selector ?? {};
+const fanoutSelector = fanoutService.spec?.selector ?? {};
+const fanoutMatchLabels = fanoutDeployment.spec?.selector?.matchLabels ?? {};
+const fanoutPodLabels = fanoutDeployment.spec?.template?.metadata?.labels ?? {};
+const fanoutMetadataLabels = fanoutDeployment.metadata?.labels ?? {};
+
+if (isSubset(mainSelector, fanoutPodLabels)) {
+  fail('main Service selector matches fanout pod labels');
+}
+if (!isSubset(fanoutSelector, fanoutPodLabels)) {
+  fail('fanout Service selector does not match fanout pod labels');
+}
+if (!isSubset(fanoutMatchLabels, fanoutPodLabels)) {
+  fail('fanout Deployment selector is not a subset of pod labels');
+}
+if (mainSelector['app.kubernetes.io/name'] === fanoutSelector['app.kubernetes.io/name']) {
+  fail('main and fanout Services share app.kubernetes.io/name selectors');
+}
+if (fanoutMetadataLabels['app.kubernetes.io/name'] !== fanoutSelector['app.kubernetes.io/name']) {
+  fail('fanout Deployment metadata labels do not use fanout app name');
+}
+if (envValue(fanoutContainer.env, 'LANGFUSE_FANOUT_REDIS_URI') !== 'redis://langfuse-fanout-redis:6379') {
+  fail('fanout Deployment did not render configured Redis URI');
+}
+if (
+  envValue(fanoutContainer.env, 'LANGFUSE_FANOUT_PUBLIC_URL') !==
+  'http://librechat-librechat-langfuse-fanout.default.svc.cluster.local:4318'
+) {
+  fail('fanout Deployment did not render derived public URL');
+}
+if (fanoutContainer.livenessProbe?.httpGet?.path !== '/healthz') {
+  fail('fanout Deployment missing /healthz liveness probe');
+}
+if (fanoutContainer.readinessProbe?.httpGet?.path !== '/healthz') {
+  fail('fanout Deployment missing /healthz readiness probe');
+}
+
+console.log('PASS: Langfuse fanout selectors are isolated from the main LibreChat Service');
+NODE
--- a/helm/librechat/values.yaml
+++ b/helm/librechat/values.yaml
@ -284,6 +284,77 @@ dnsConfig: {}
 updateStrategy:
  type: RollingUpdate

+langfuseFanout:
+  enabled: false
+  replicaCount: 1
+  image:
+    repository: librechat-langfuse-fanout
+    tag: "latest"
+    pullPolicy: IfNotPresent
+  otelCollector:
+    receiverEndpoint: 127.0.0.1:4319
+    image:
+      repository: otel/opentelemetry-collector-contrib
+      tag: "0.143.0"
+      pullPolicy: IfNotPresent
+    resources: {}
+  service:
+    type: ClusterIP
+    port: 4318
+    annotations: {}
+  central:
+    baseUrl: https://cloud.langfuse.com
+    authHeaderSecret:
+      name: ""
+      key: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER
+  metrics:
+    secret:
+      # Optional bearer token secret for scraping the gateway's /metrics endpoint.
+      # When omitted, /metrics returns 401.
+      name: ""
+      key: METRICS_SECRET
+  redis:
+    # Redis stores short-lived Langfuse media upload plans so multiple gateway
+    # replicas can handle create/upload requests. If empty and redis.enabled is
+    # true, the chart derives the bundled Redis service URI.
+    uri: ""
+    username: ""
+    passwordSecret:
+      name: ""
+      key: REDIS_PASSWORD
+    keyPrefix: langfuse-fanout
+  tenant:
+    # Destination map keys are emitted by LibreChat as trace attributes and
+    # matched by the gateway. Use lowercase keys matching ^[a-z][a-z0-9_-]*$.
+    destinations:
+      eu:
+        baseUrl: https://cloud.langfuse.com
+      us:
+        baseUrl: https://us.cloud.langfuse.com
+      jp:
+        baseUrl: https://jp.cloud.langfuse.com
+  upstreamTimeout: 30s
+  # Optional override for one-time media upload URLs returned by the gateway.
+  # When empty, the chart derives the internal fanout Service URL.
+  publicUrl: ""
+  traceCollectorUrl: http://127.0.0.1:4319
+  memoryLimitMiB: 256
+  memorySpikeLimitMiB: 64
+  batchTimeout: 1s
+  batchSendSize: 128
+  metadataCardinalityLimit: 1000
+  livenessProbe:
+    httpGet:
+      path: /healthz
+      port: 4318
+  readinessProbe:
+    httpGet:
+      path: /healthz
+      port: 4318
+  resources: {}
+  podAnnotations: {}
+  podLabels: {}
+
 # Extra ConfigMaps to be created alongside the main ones
 additionalConfigMaps: {}
 #   custom: # suffix of the ConfigMap name