From 6ff3dcd4ee7dddffb0c4e3c10e990b57d5a8fb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Dunglas?= Date: Mon, 9 Nov 2026 13:00:14 +0100 Subject: [PATCH] logging: bound OTLP provider shutdown Releasing the last OTLP core held the provider mutex while flushing the exporter with no deadline, so a stalled collector could hang config unloading and block the next config from acquiring a provider. --- modules/logging/otlpcore.go | 16 +++-- modules/logging/otlpcore_test.go | 108 +++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/modules/logging/otlpcore.go b/modules/logging/otlpcore.go index 46702b6bb..25d550c10 100644 --- a/modules/logging/otlpcore.go +++ b/modules/logging/otlpcore.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "sync" + "time" "go.opentelemetry.io/contrib/bridges/otelzap" "go.opentelemetry.io/contrib/exporters/autoexport" @@ -195,13 +196,15 @@ func (c *otlpCore) prepare(fields []zapcore.Field) []zapcore.Field { return prepared } -var globalLoggerProvider loggerProvider +var globalLoggerProvider = loggerProvider{shutdownTimeout: 5 * time.Second} // loggerProvider shares one exporter between all logs and across config reloads. type loggerProvider struct { mu sync.Mutex provider *sdklog.LoggerProvider users int + // bounds the flush so a stalled exporter cannot hang config unloading + shutdownTimeout time.Duration } func (p *loggerProvider) acquire(ctx context.Context) (*sdklog.LoggerProvider, error) { @@ -235,16 +238,19 @@ func (p *loggerProvider) acquire(ctx context.Context) (*sdklog.LoggerProvider, e func (p *loggerProvider) release() error { p.mu.Lock() - defer p.mu.Unlock() - p.users-- if p.users > 0 { + p.mu.Unlock() return nil } - provider := p.provider p.provider = nil - if err := provider.Shutdown(context.Background()); err != nil { + timeout := p.shutdownTimeout + p.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + if err := provider.Shutdown(ctx); err != nil { return fmt.Errorf("shutting down OTLP logger provider: %w", err) } diff --git a/modules/logging/otlpcore_test.go b/modules/logging/otlpcore_test.go index 371980e48..ff3eb2d4f 100644 --- a/modules/logging/otlpcore_test.go +++ b/modules/logging/otlpcore_test.go @@ -17,13 +17,18 @@ package logging import ( "context" "encoding/hex" + "errors" "io" "net/http" "net/http/httptest" "reflect" "sync" + "sync/atomic" "testing" + "time" + "go.opentelemetry.io/contrib/exporters/autoexport" + sdklog "go.opentelemetry.io/otel/sdk/log" collogspb "go.opentelemetry.io/proto/otlp/collector/logs/v1" commonpb "go.opentelemetry.io/proto/otlp/common/v1" logspb "go.opentelemetry.io/proto/otlp/logs/v1" @@ -209,3 +214,106 @@ func TestOTLPCoreLinksRecordsToSpans(t *testing.T) { t.Errorf("expected no trace ID for invalid IDs, got %x", untraced.GetTraceId()) } } + +// stalledExporter hangs on shutdown, ignoring its context, until unblocked. +type stalledExporter struct { + shuttingDown chan struct{} + unblock func() + unblocked chan struct{} +} + +func (*stalledExporter) Export(context.Context, []sdklog.Record) error { return nil } +func (*stalledExporter) ForceFlush(context.Context) error { return nil } + +func (e *stalledExporter) Shutdown(context.Context) error { + e.shuttingDown <- struct{}{} + <-e.unblocked + return nil +} + +var currentStalledExporter atomic.Pointer[stalledExporter] + +func init() { + autoexport.RegisterLogExporter("caddy-test-stalled", func(context.Context) (sdklog.Exporter, error) { + return currentStalledExporter.Load(), nil + }) +} + +func useStalledExporter(t *testing.T, shutdownTimeout time.Duration) *stalledExporter { + t.Helper() + + unblocked := make(chan struct{}) + e := &stalledExporter{ + shuttingDown: make(chan struct{}, 2), + unblock: sync.OnceFunc(func() { close(unblocked) }), + unblocked: unblocked, + } + currentStalledExporter.Store(e) + t.Setenv("OTEL_LOGS_EXPORTER", "caddy-test-stalled") + + globalLoggerProvider.mu.Lock() + orig := globalLoggerProvider.shutdownTimeout + globalLoggerProvider.shutdownTimeout = shutdownTimeout + globalLoggerProvider.mu.Unlock() + t.Cleanup(func() { + globalLoggerProvider.mu.Lock() + globalLoggerProvider.shutdownTimeout = orig + globalLoggerProvider.mu.Unlock() + }) + // runs first, so a failed test cannot leave the provider mutex locked + t.Cleanup(e.unblock) + + return e +} + +func TestOTLPCoreCleanupBoundsStalledExporter(t *testing.T) { + useStalledExporter(t, 10*time.Millisecond) + core := provisionOTLPCore(t) + + done := make(chan error, 1) + go func() { done <- core.Cleanup() }() + + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected the shutdown to time out, got %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("cleanup hung on a stalled exporter") + } +} + +func TestOTLPCoreAcquireDuringStalledShutdown(t *testing.T) { + exporter := useStalledExporter(t, time.Minute) + oldCore := provisionOTLPCore(t) + + done := make(chan error, 1) + go func() { done <- oldCore.Cleanup() }() + <-exporter.shuttingDown + + acquired := make(chan *OTLPCore, 1) + go func() { + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) + defer cancel() + newCore := new(OTLPCore) + if err := newCore.Provision(ctx); err != nil { + t.Error(err) + } + acquired <- newCore + }() + + var newCore *OTLPCore + select { + case newCore = <-acquired: + case <-time.After(5 * time.Second): + t.Fatal("provisioning waited for the previous provider to shut down") + } + + exporter.unblock() + if err := <-done; err != nil { + t.Fatal(err) + } + if err := newCore.Cleanup(); err != nil { + t.Fatal(err) + } +}