From ab7e6b6d81cfc42eaabd5ffb172fd912bb7624cf Mon Sep 17 00:00:00 2001 From: Zen Dodd Date: Sat, 3 Oct 2026 12:10:58 +1000 Subject: [PATCH] caddyhttp: end request-body deadline ownership with handler (#8107) Co-authored-by: Claude Opus 5.5 Co-authored-by: Francis Lavoie --- modules/caddyhttp/idletimeout.go | 81 +++++- modules/caddyhttp/idletimeout_test.go | 292 ++++++++++++++++++++ modules/caddyhttp/server.go | 7 +- modules/caddyhttp/timeouts/timeouts.go | 7 +- modules/caddyhttp/timeouts/timeouts_test.go | 42 +++ 5 files changed, 418 insertions(+), 11 deletions(-) diff --git a/modules/caddyhttp/idletimeout.go b/modules/caddyhttp/idletimeout.go index 0997ddc6e..8bfa8ac89 100644 --- a/modules/caddyhttp/idletimeout.go +++ b/modules/caddyhttp/idletimeout.go @@ -17,6 +17,7 @@ package caddyhttp import ( "io" "net/http" + "sync" "time" "go.uber.org/zap" @@ -79,25 +80,91 @@ type IdleTimeoutReader struct { Deadline IdleDeadline Logger *zap.Logger + // DrainDeadline makes HandlerDone arm a final idle deadline if + // the body is unfinished, bounding net/http's post-handler drain. + // Only HTTP/1 drains a request body on the connection after the + // handler returns, so it is pointless (and costs a timer on + // HTTP/2) for other protocols or requests without a body. + DrainDeadline bool + + mu sync.Mutex unsupported bool + deadlineSet bool + finished bool + terminalErr error } func (r *IdleTimeoutReader) Read(p []byte) (int, error) { - if !r.unsupported { - if err := r.Ctrl.SetReadDeadline(r.Deadline.next()); err != nil { - r.unsupported = true - if c := r.Logger.Check(zapcore.DebugLevel, "could not set read deadline"); c != nil { - c.Write(zap.Error(err)) - } - } + r.mu.Lock() + if r.terminalErr != nil { + err := r.terminalErr + r.mu.Unlock() + return 0, err } + if !r.finished && !r.unsupported { + r.setDeadlineLocked("could not set read deadline") + } + r.mu.Unlock() n, err := r.ReadCloser.Read(p) + + r.mu.Lock() r.Deadline.transferred += int64(n) + if err != nil { + r.terminalErr = err + if !r.finished { + r.clearDeadlineLocked() + } + } + r.mu.Unlock() return n, err } +// HandlerDone prevents later body reads from using the response controller. +// If the body is not finished and DrainDeadline is set, it leaves an idle +// deadline armed for net/http's post-handler drain. It must be called before +// the installing handler returns. +func (r *IdleTimeoutReader) HandlerDone() { + r.mu.Lock() + defer r.mu.Unlock() + + if r.finished { + return + } + if r.terminalErr != nil { + r.clearDeadlineLocked() + } else if r.DrainDeadline && !r.deadlineSet && !r.unsupported { + r.setDeadlineLocked("could not set final read deadline") + } + r.finished = true +} + +func (r *IdleTimeoutReader) setDeadlineLocked(logMessage string) { + if err := r.Ctrl.SetReadDeadline(r.Deadline.next()); err != nil { + r.unsupported = true + if c := r.Logger.Check(zapcore.DebugLevel, logMessage); c != nil { + c.Write(zap.Error(err)) + } + return + } + r.deadlineSet = true +} + +func (r *IdleTimeoutReader) clearDeadlineLocked() { + if !r.deadlineSet { + return + } + if err := r.Ctrl.SetReadDeadline(time.Time{}); err != nil { + r.unsupported = true + if c := r.Logger.Check(zapcore.DebugLevel, "could not clear read deadline"); c != nil { + c.Write(zap.Error(err)) + } + return + } + r.deadlineSet = false +} + // IdleTimeoutWriter wraps a ResponseWriter with IdleDeadline, resetting // the write deadline before every Write call, the same way // IdleTimeoutReader does for reads. A handler that pauses between diff --git a/modules/caddyhttp/idletimeout_test.go b/modules/caddyhttp/idletimeout_test.go index 7390244cd..703ec233d 100644 --- a/modules/caddyhttp/idletimeout_test.go +++ b/modules/caddyhttp/idletimeout_test.go @@ -16,9 +16,13 @@ package caddyhttp import ( "bytes" + "errors" + "fmt" "io" "net/http" "net/http/httptest" + "strings" + "sync" "testing" "time" @@ -27,6 +31,43 @@ import ( "go.uber.org/zap" ) +type readDeadlineRecorder struct { + *httptest.ResponseRecorder + mu sync.Mutex + deadlines []time.Time + failAt int +} + +func (w *readDeadlineRecorder) SetReadDeadline(deadline time.Time) error { + w.mu.Lock() + defer w.mu.Unlock() + w.deadlines = append(w.deadlines, deadline) + if len(w.deadlines) == w.failAt { + return errors.New("deadline failure") + } + return nil +} + +func (w *readDeadlineRecorder) snapshot() []time.Time { + w.mu.Lock() + defer w.mu.Unlock() + return append([]time.Time(nil), w.deadlines...) +} + +type blockingReadCloser struct { + started chan struct{} + release chan struct{} + once sync.Once +} + +func (r *blockingReadCloser) Read([]byte) (int, error) { + r.once.Do(func() { close(r.started) }) + <-r.release + return 0, io.EOF +} + +func (*blockingReadCloser) Close() error { return nil } + // pacedReader emits chunkCount chunks of chunkSize bytes, sleeping delay // before each one, simulating a client that trickles a request body. type pacedReader struct { @@ -87,6 +128,257 @@ func TestIdleTimeoutReader(t *testing.T) { }) } +func TestIdleTimeoutReaderClearsDeadlineAtEOF(t *testing.T) { + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("body")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + } + + _, err := io.Copy(io.Discard, r) + require.NoError(t, err) + deadlines := w.snapshot() + require.NotEmpty(t, deadlines) + assert.True(t, deadlines[len(deadlines)-1].IsZero()) +} + +func TestIdleTimeoutReaderHandlerDone(t *testing.T) { + body := &blockingReadCloser{ + started: make(chan struct{}), + release: make(chan struct{}), + } + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + } + + readDone := make(chan struct{}) + go func() { + defer close(readDone) + _, _ = r.Read(make([]byte, 1)) + }() + <-body.started + + r.HandlerDone() + deadlines := w.snapshot() + require.Len(t, deadlines, 1) + assert.False(t, deadlines[0].IsZero()) + + close(body.release) + <-readDone + _, _ = r.Read(make([]byte, 1)) + r.HandlerDone() + assert.Len(t, w.snapshot(), len(deadlines)) +} + +func TestIdleTimeoutReaderHandlerDoneAfterResetFailure(t *testing.T) { + w := &readDeadlineRecorder{ + ResponseRecorder: httptest.NewRecorder(), + failAt: 2, + } + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("body")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + } + + _, _ = r.Read(make([]byte, 1)) + _, _ = r.Read(make([]byte, 1)) + r.HandlerDone() + + deadlines := w.snapshot() + require.Len(t, deadlines, 2) + assert.False(t, deadlines[0].IsZero()) + assert.False(t, deadlines[1].IsZero()) +} + +func TestIdleTimeoutReaderHandlerDoneRetriesTerminalClear(t *testing.T) { + w := &readDeadlineRecorder{ + ResponseRecorder: httptest.NewRecorder(), + failAt: 3, + } + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("body")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + } + + _, err := io.Copy(io.Discard, r) + require.NoError(t, err) + r.HandlerDone() + + deadlines := w.snapshot() + require.Len(t, deadlines, 4) + assert.False(t, deadlines[0].IsZero()) + assert.False(t, deadlines[1].IsZero()) + assert.True(t, deadlines[2].IsZero()) + assert.True(t, deadlines[3].IsZero()) +} + +func TestIdleTimeoutReaderHandlerDoneArmsDrain(t *testing.T) { + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("body")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + DrainDeadline: true, + } + + r.HandlerDone() + deadlines := w.snapshot() + require.Len(t, deadlines, 1) + assert.False(t, deadlines[0].IsZero()) +} + +func TestIdleTimeoutReaderHandlerDoneWithoutDrainDeadline(t *testing.T) { + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("body")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + } + + r.HandlerDone() + assert.Empty(t, w.snapshot()) +} + +func TestIdleTimeoutReaderAfterHandlerReturnsHTTP2(t *testing.T) { + lateBody := make(chan io.ReadCloser, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wrapped := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Second}, + Logger: zap.NewNop(), + } + defer wrapped.HandlerDone() + lateBody <- wrapped + _, _ = io.WriteString(w, "ok") + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + resp, err := srv.Client().Post(srv.URL, "text/plain", strings.NewReader("body")) + require.NoError(t, err) + responseBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, 2, resp.ProtoMajor) + require.Equal(t, "ok", string(responseBody)) + + readResult := make(chan error, 1) + go func() { + var result error + defer func() { + if recovered := recover(); recovered != nil { + result = fmt.Errorf("late request-body read panicked: %v", recovered) + } + readResult <- result + }() + _, _ = io.Copy(io.Discard, <-lateBody) + }() + select { + case err := <-readResult: + assert.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("late request-body read did not return") + } +} + +func TestIdleTimeoutReaderHandlerDonePreservesHTTP1Connection(t *testing.T) { + const idle = 100 * time.Millisecond + + remoteAddresses := make(chan string, 2) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wrapped := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: idle}, + Logger: zap.NewNop(), + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + } + defer wrapped.HandlerDone() + remoteAddresses <- r.RemoteAddr + if r.Method == http.MethodPost { + _, _ = wrapped.Read(make([]byte, 1)) + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + transport := &http.Transport{MaxConnsPerHost: 1} + client := &http.Client{Transport: transport} + defer transport.CloseIdleConnections() + + resp, err := client.Post(srv.URL, "text/plain", strings.NewReader("body")) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + firstAddress := <-remoteAddresses + + time.Sleep(3 * idle) + resp, err = client.Get(srv.URL) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + assert.Equal(t, firstAddress, <-remoteAddresses) +} + +func TestIdleTimeoutReaderDeadlineClearedAfterBodyEOF(t *testing.T) { + const idle = 150 * time.Millisecond + + upstream := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + })) + defer upstream.Close() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wrapped := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: idle}, + Logger: zap.NewNop(), + } + defer wrapped.HandlerDone() + + upReq, err := http.NewRequestWithContext(r.Context(), r.Method, upstream.URL, wrapped) + if err != nil { + t.Error(err) + return + } + upReq.ContentLength = r.ContentLength + resp, err := http.DefaultClient.Do(upReq) + if err != nil { + t.Error(err) + return + } + _ = resp.Body.Close() + + select { + case <-r.Context().Done(): + w.WriteHeader(http.StatusTeapot) + case <-time.After(3 * idle): + w.WriteHeader(http.StatusOK) + } + })) + defer srv.Close() + + resp, err := http.Post(srv.URL, "application/octet-stream", strings.NewReader("{}")) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + func TestIdleTimeoutReader_HardDeadlineCapsIdleReset(t *testing.T) { const idle = 500 * time.Millisecond const hard = 200 * time.Millisecond diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index 87bb6232f..e04d9d983 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -586,7 +586,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { writeHardDeadline = start.Add(time.Duration(s.WriteTimeout)) } if s.ReadIdleTimeout > 0 && r.Body != nil { - r.Body = &IdleTimeoutReader{ + idleReader := &IdleTimeoutReader{ ReadCloser: r.Body, Ctrl: rc, Deadline: IdleDeadline{ @@ -595,8 +595,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { MinRate: s.ReadMinRate, HardDeadline: readHardDeadline, }, - Logger: s.logger, + Logger: s.logger, + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, } + defer idleReader.HandlerDone() + r.Body = idleReader } if s.WriteIdleTimeout > 0 { w = &IdleTimeoutWriter{ diff --git a/modules/caddyhttp/timeouts/timeouts.go b/modules/caddyhttp/timeouts/timeouts.go index 7e454c6f8..de41e88f5 100644 --- a/modules/caddyhttp/timeouts/timeouts.go +++ b/modules/caddyhttp/timeouts/timeouts.go @@ -92,7 +92,7 @@ func (t Timeouts) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht start := time.Now() if t.ReadTimeout > 0 && r.Body != nil { - r.Body = &caddyhttp.IdleTimeoutReader{ + idleReader := &caddyhttp.IdleTimeoutReader{ ReadCloser: r.Body, Ctrl: rc, Deadline: caddyhttp.IdleDeadline{ @@ -100,8 +100,11 @@ func (t Timeouts) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht Timeout: t.ReadTimeout, MinRate: t.ReadMinRate, }, - Logger: t.logger, + Logger: t.logger, + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, } + defer idleReader.HandlerDone() + r.Body = idleReader } if t.WriteTimeout > 0 { w = &caddyhttp.IdleTimeoutWriter{ diff --git a/modules/caddyhttp/timeouts/timeouts_test.go b/modules/caddyhttp/timeouts/timeouts_test.go index 840b00c17..d4561fdf1 100644 --- a/modules/caddyhttp/timeouts/timeouts_test.go +++ b/modules/caddyhttp/timeouts/timeouts_test.go @@ -18,6 +18,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -28,6 +29,16 @@ import ( "github.com/caddyserver/caddy/v2/modules/caddyhttp" ) +type readDeadlineRecorder struct { + *httptest.ResponseRecorder + deadlines []time.Time +} + +func (w *readDeadlineRecorder) SetReadDeadline(deadline time.Time) error { + w.deadlines = append(w.deadlines, deadline) + return nil +} + // pacedReader emits chunkCount chunks of chunkSize bytes, sleeping delay // before each one, simulating a client that trickles a request body. type pacedReader struct { @@ -83,6 +94,37 @@ func TestTimeouts_ReadTimeoutIsIdleReset(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } +func TestTimeoutsStopsReadDeadlineBeforeReturning(t *testing.T) { + tm := Timeouts{ReadTimeout: time.Second, logger: zap.NewNop()} + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("body")) + var body io.Reader + + err := tm.ServeHTTP(w, req, caddyhttp.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) error { + body = r.Body + _, _ = body.Read(make([]byte, 1)) + return nil + })) + require.NoError(t, err) + require.Len(t, w.deadlines, 1) + assert.False(t, w.deadlines[0].IsZero()) + + _, _ = io.Copy(io.Discard, body) + assert.Len(t, w.deadlines, 1) +} + +func TestTimeoutsSkipsDrainDeadlineWithoutBody(t *testing.T) { + tm := Timeouts{ReadTimeout: time.Second, logger: zap.NewNop()} + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + req := httptest.NewRequest(http.MethodGet, "/", nil) + + err := tm.ServeHTTP(w, req, caddyhttp.HandlerFunc(func(http.ResponseWriter, *http.Request) error { + return nil + })) + require.NoError(t, err) + assert.Empty(t, w.deadlines) +} + func TestTimeouts_WriteMaxChunkOverride(t *testing.T) { const size = 10000 const maxChunk = 100