mirror of
https://github.com/caddyserver/caddy.git
synced 2026-11-16 21:16:00 +00:00
logging: bound OTLP provider shutdown
Releasing the last OTLP core held the provider mutex while flushing the exporter with no deadline, so a stalled collector could hang config unloading and block the next config from acquiring a provider.
This commit is contained in:
parent
ac13f0098f
commit
6ff3dcd4ee
2 changed files with 119 additions and 5 deletions
|
|
@ -18,6 +18,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/contrib/bridges/otelzap"
|
||||
"go.opentelemetry.io/contrib/exporters/autoexport"
|
||||
|
|
@ -195,13 +196,15 @@ func (c *otlpCore) prepare(fields []zapcore.Field) []zapcore.Field {
|
|||
return prepared
|
||||
}
|
||||
|
||||
var globalLoggerProvider loggerProvider
|
||||
var globalLoggerProvider = loggerProvider{shutdownTimeout: 5 * time.Second}
|
||||
|
||||
// loggerProvider shares one exporter between all logs and across config reloads.
|
||||
type loggerProvider struct {
|
||||
mu sync.Mutex
|
||||
provider *sdklog.LoggerProvider
|
||||
users int
|
||||
// bounds the flush so a stalled exporter cannot hang config unloading
|
||||
shutdownTimeout time.Duration
|
||||
}
|
||||
|
||||
func (p *loggerProvider) acquire(ctx context.Context) (*sdklog.LoggerProvider, error) {
|
||||
|
|
@ -235,16 +238,19 @@ func (p *loggerProvider) acquire(ctx context.Context) (*sdklog.LoggerProvider, e
|
|||
|
||||
func (p *loggerProvider) release() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.users--
|
||||
if p.users > 0 {
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
provider := p.provider
|
||||
p.provider = nil
|
||||
if err := provider.Shutdown(context.Background()); err != nil {
|
||||
timeout := p.shutdownTimeout
|
||||
p.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
if err := provider.Shutdown(ctx); err != nil {
|
||||
return fmt.Errorf("shutting down OTLP logger provider: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,13 +17,18 @@ package logging
|
|||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/contrib/exporters/autoexport"
|
||||
sdklog "go.opentelemetry.io/otel/sdk/log"
|
||||
collogspb "go.opentelemetry.io/proto/otlp/collector/logs/v1"
|
||||
commonpb "go.opentelemetry.io/proto/otlp/common/v1"
|
||||
logspb "go.opentelemetry.io/proto/otlp/logs/v1"
|
||||
|
|
@ -209,3 +214,106 @@ func TestOTLPCoreLinksRecordsToSpans(t *testing.T) {
|
|||
t.Errorf("expected no trace ID for invalid IDs, got %x", untraced.GetTraceId())
|
||||
}
|
||||
}
|
||||
|
||||
// stalledExporter hangs on shutdown, ignoring its context, until unblocked.
|
||||
type stalledExporter struct {
|
||||
shuttingDown chan struct{}
|
||||
unblock func()
|
||||
unblocked chan struct{}
|
||||
}
|
||||
|
||||
func (*stalledExporter) Export(context.Context, []sdklog.Record) error { return nil }
|
||||
func (*stalledExporter) ForceFlush(context.Context) error { return nil }
|
||||
|
||||
func (e *stalledExporter) Shutdown(context.Context) error {
|
||||
e.shuttingDown <- struct{}{}
|
||||
<-e.unblocked
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentStalledExporter atomic.Pointer[stalledExporter]
|
||||
|
||||
func init() {
|
||||
autoexport.RegisterLogExporter("caddy-test-stalled", func(context.Context) (sdklog.Exporter, error) {
|
||||
return currentStalledExporter.Load(), nil
|
||||
})
|
||||
}
|
||||
|
||||
func useStalledExporter(t *testing.T, shutdownTimeout time.Duration) *stalledExporter {
|
||||
t.Helper()
|
||||
|
||||
unblocked := make(chan struct{})
|
||||
e := &stalledExporter{
|
||||
shuttingDown: make(chan struct{}, 2),
|
||||
unblock: sync.OnceFunc(func() { close(unblocked) }),
|
||||
unblocked: unblocked,
|
||||
}
|
||||
currentStalledExporter.Store(e)
|
||||
t.Setenv("OTEL_LOGS_EXPORTER", "caddy-test-stalled")
|
||||
|
||||
globalLoggerProvider.mu.Lock()
|
||||
orig := globalLoggerProvider.shutdownTimeout
|
||||
globalLoggerProvider.shutdownTimeout = shutdownTimeout
|
||||
globalLoggerProvider.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalLoggerProvider.mu.Lock()
|
||||
globalLoggerProvider.shutdownTimeout = orig
|
||||
globalLoggerProvider.mu.Unlock()
|
||||
})
|
||||
// runs first, so a failed test cannot leave the provider mutex locked
|
||||
t.Cleanup(e.unblock)
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
func TestOTLPCoreCleanupBoundsStalledExporter(t *testing.T) {
|
||||
useStalledExporter(t, 10*time.Millisecond)
|
||||
core := provisionOTLPCore(t)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- core.Cleanup() }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("expected the shutdown to time out, got %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup hung on a stalled exporter")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOTLPCoreAcquireDuringStalledShutdown(t *testing.T) {
|
||||
exporter := useStalledExporter(t, time.Minute)
|
||||
oldCore := provisionOTLPCore(t)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- oldCore.Cleanup() }()
|
||||
<-exporter.shuttingDown
|
||||
|
||||
acquired := make(chan *OTLPCore, 1)
|
||||
go func() {
|
||||
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
|
||||
defer cancel()
|
||||
newCore := new(OTLPCore)
|
||||
if err := newCore.Provision(ctx); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
acquired <- newCore
|
||||
}()
|
||||
|
||||
var newCore *OTLPCore
|
||||
select {
|
||||
case newCore = <-acquired:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("provisioning waited for the previous provider to shut down")
|
||||
}
|
||||
|
||||
exporter.unblock()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := newCore.Cleanup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue