From 30d3bd2a8959d4d00ca72446f23a59a09f02a83b Mon Sep 17 00:00:00 2001 From: Francis Lavoie Date: Mon, 5 Oct 2026 17:47:14 -0400 Subject: [PATCH] caddyhttp: only bound HTTP/2 body reads in flight with the idle timeout (#8125) * caddyhttp: only bound HTTP/2 body reads in flight with the idle timeout On HTTP/2, the read deadline is a timer that fails the request body when it fires, even with no read in flight. IdleTimeoutReader left the idle deadline armed between reads, so a handler that stopped reading for longer than read_body_idle (1 minute by default) lost the body, although the client was only held back by flow control. For example, reverse_proxy uploading to an upstream that's slow to accept the body failed with a 502. This is the read side of #8118. Like IdleTimeoutWriter does since read on HTTP/2, so the idle deadline only bounds reads in flight. A client that stops sending is still cut off, since the deadline is set before each read. HTTP/1 and HTTP/3 only check the deadline during a read, so they're unchanged. Written with Claude Code (Claude Opus 5.5); the human submitter is responsible for reviewing it. Co-Authored-By: Claude Opus 5.5 * caddyhttp: track a released hard read deadline as armed From review: with HardDeadline set, releasing the idle deadline between reads left an HTTP/2 timer armed (at HardDeadline) while deadlineSet recorded that none was, so clearDeadlineLocked would skip clearing it. Track which deadline is armed (none, idle, or only the hard one) instead of a bool, and have each check ask the question it means: re-arming and releasing only apply to an armed idle deadline, while clearing applies to any. The writer's deadlineSet already only means "idle deadline armed", since nothing clears the hard deadline it leaves; document that. Written with Claude Code (Claude Opus 5.5); the human submitter is responsible for reviewing it. Co-Authored-By: Claude Opus 5.5 * caddyhttp: clear released hard deadline at handler end, skip no-op sets From review: - After a partial read releases the idle deadline to HardDeadline, a handler that returns without reading again left that timer armed past HandlerDone, which only cleared after a terminal error. HandlerDone now also clears a released hard deadline. - When the idle deadline is capped to HardDeadline, the reader and writer set the same deadline again when releasing it, and the writer once more in HandlerDone. On HTTP/2 each is a message to the serve loop. Both now remember the deadline they armed and skip setting it again. Within a request, only these wrappers set the client's deadlines. Replaces the earlier test that faked a terminal error with the partial read followed directly by HandlerDone. Written with Claude Code (Claude Opus 5.5); the human submitter is responsible for reviewing it. Co-Authored-By: Claude Opus 5.5 * caddyhttp: only skip re-setting an armed hard deadline Skipping any deadline equal to the one already armed also skipped idle deadlines on Windows, where two calls to time.Now in the same clock tick return the same time, so successive idle deadlines can be equal. That broke tests counting deadline calls there. Only skip HardDeadline, the case from review where the idle deadline is capped to it; other deadlines are always set. Written with Claude Code (Claude Opus 5.5); the human submitter is responsible for reviewing it. Co-Authored-By: Claude Opus 5.5 * caddyhttp: restore deadlines without trusting cached transport state Always set the requested read and write deadlines because another handler can change them through its own response controller. Remove the armedAt shortcut and cover intervening deadline changes for reads, writes, ReadFrom, flushes, and HandlerDone. AI-assisted with OpenAI Codex (GPT-6); tested under human direction. --------- Co-authored-by: Claude Opus 5.5 --- modules/caddyhttp/idletimeout.go | 110 ++++++++--- modules/caddyhttp/idletimeout_test.go | 247 ++++++++++++++++++++++++- modules/caddyhttp/server.go | 5 +- modules/caddyhttp/timeouts/timeouts.go | 5 +- 4 files changed, 338 insertions(+), 29 deletions(-) diff --git a/modules/caddyhttp/idletimeout.go b/modules/caddyhttp/idletimeout.go index 976580919..cd4fd4f70 100644 --- a/modules/caddyhttp/idletimeout.go +++ b/modules/caddyhttp/idletimeout.go @@ -122,13 +122,38 @@ type IdleTimeoutReader struct { // HTTP/2) for other protocols or requests without a body. DrainDeadline bool + // ClearBetweenReads makes the reader put back HardDeadline (no + // deadline unless a hard ceiling is configured) after every + // successful read, so the idle deadline only bounds reads in + // flight. It is required for HTTP/2, where the read deadline is a + // timer that fails the body when it fires, even with no read in + // flight: otherwise a handler that stops reading for longer than the + // timeout (e.g. while an upstream is slow to accept the body) loses + // the body, although the client is only held back by flow control. + // Other protocols only check the deadline during a read. + ClearBetweenReads bool + mu sync.Mutex unsupported bool - deadlineSet bool + armed armedDeadline finished bool terminalErr error } +// armedDeadline is which read deadline IdleTimeoutReader has armed. +type armedDeadline uint8 + +const ( + // deadlineNone means no deadline is armed. + deadlineNone armedDeadline = iota + // deadlineIdle means the idle deadline from IdleDeadline.next is + // armed. + deadlineIdle + // deadlineHard means only HardDeadline is armed, as put back + // between reads with ClearBetweenReads. + deadlineHard +) + func (r *IdleTimeoutReader) Read(p []byte) (int, error) { r.mu.Lock() if r.terminalErr != nil { @@ -150,6 +175,8 @@ func (r *IdleTimeoutReader) Read(p []byte) (int, error) { if !r.finished { r.clearDeadlineLocked() } + } else if r.ClearBetweenReads && !r.finished { + r.releaseDeadlineLocked() } r.mu.Unlock() @@ -158,8 +185,9 @@ func (r *IdleTimeoutReader) Read(p []byte) (int, error) { // HandlerDone prevents later body reads from using the response controller. // If the body is not finished and DrainDeadline is set, it leaves an idle -// deadline armed for net/http's post-handler drain. It must be called before -// the installing handler returns. +// deadline armed for net/http's post-handler drain. Otherwise, it clears a +// deadline left from a finished body or put back between reads. It must be +// called before the installing handler returns. func (r *IdleTimeoutReader) HandlerDone() { r.mu.Lock() defer r.mu.Unlock() @@ -167,9 +195,9 @@ func (r *IdleTimeoutReader) HandlerDone() { if r.finished { return } - if r.terminalErr != nil { + if r.terminalErr != nil || r.armed == deadlineHard { r.clearDeadlineLocked() - } else if r.DrainDeadline && !r.deadlineSet && !r.unsupported { + } else if r.DrainDeadline && r.armed != deadlineIdle && !r.unsupported { r.setDeadlineLocked("could not set final read deadline") } r.finished = true @@ -199,34 +227,61 @@ func (r *IdleTimeoutReader) Override(timeout time.Duration, minRate int64) (rest // rearmLocked moves an armed deadline to what the current settings give. func (r *IdleTimeoutReader) rearmLocked() { - if r.deadlineSet && !r.finished && !r.unsupported { + if r.armed == deadlineIdle && !r.finished && !r.unsupported { r.setDeadlineLocked("could not set read deadline") } } func (r *IdleTimeoutReader) setDeadlineLocked(logMessage string) { - if err := r.Ctrl.SetReadDeadline(r.Deadline.next()); err != nil { + if r.setReadDeadlineLocked(r.Deadline.next(), logMessage) { + r.armed = deadlineIdle + } +} + +// setReadDeadlineLocked always sets the requested deadline: another handler +// may have changed it through its own response controller since our last call. +// It reports whether the deadline was set successfully. +func (r *IdleTimeoutReader) setReadDeadlineLocked(deadline time.Time, logMessage string) bool { + if err := r.Ctrl.SetReadDeadline(deadline); err != nil { r.unsupported = true if c := r.Logger.Check(zapcore.DebugLevel, logMessage); c != nil { c.Write(zap.Error(err)) } + return false + } + return true +} + +// releaseDeadlineLocked replaces the idle deadline with HardDeadline. +func (r *IdleTimeoutReader) releaseDeadlineLocked() { + if r.armed != deadlineIdle || r.unsupported { return } - r.deadlineSet = true + + // once the hard deadline has passed, the deadline set before the + // read (capped to it) has already expired; keep it + if !r.Deadline.HardDeadline.IsZero() && !time.Now().Before(r.Deadline.HardDeadline) { + return + } + + if !r.setReadDeadlineLocked(r.Deadline.HardDeadline, "could not release read deadline") { + return + } + if r.Deadline.HardDeadline.IsZero() { + r.armed = deadlineNone + } else { + r.armed = deadlineHard + } } func (r *IdleTimeoutReader) clearDeadlineLocked() { - if !r.deadlineSet { + if r.armed == deadlineNone { return } - if err := r.Ctrl.SetReadDeadline(time.Time{}); err != nil { - r.unsupported = true - if c := r.Logger.Check(zapcore.DebugLevel, "could not clear read deadline"); c != nil { - c.Write(zap.Error(err)) - } + if !r.setReadDeadlineLocked(time.Time{}, "could not clear read deadline") { return } - r.deadlineSet = false + r.armed = deadlineNone } // IdleTimeoutWriter wraps a ResponseWriter with IdleDeadline, resetting @@ -270,6 +325,8 @@ type IdleTimeoutWriter struct { ClearBetweenWrites bool unsupported bool + // deadlineSet is whether the idle deadline is armed. clearDeadline + // leaves HardDeadline armed, if any, which nothing needs to undo. deadlineSet bool unflushed bool } @@ -279,14 +336,23 @@ func (w *IdleTimeoutWriter) resetDeadline() { return } - if err := w.Ctrl.SetWriteDeadline(w.Deadline.next()); err != nil { + if w.setWriteDeadline(w.Deadline.next(), "could not set write deadline") { + w.deadlineSet = true + } +} + +// setWriteDeadline always sets the requested deadline: another handler may +// have changed it through its own response controller since our last call. +// It reports whether the deadline was set successfully. +func (w *IdleTimeoutWriter) setWriteDeadline(deadline time.Time, logMessage string) bool { + if err := w.Ctrl.SetWriteDeadline(deadline); err != nil { w.unsupported = true - if c := w.Logger.Check(zapcore.DebugLevel, "could not set write deadline"); c != nil { + if c := w.Logger.Check(zapcore.DebugLevel, logMessage); c != nil { c.Write(zap.Error(err)) } - return + return false } - w.deadlineSet = true + return true } // clearDeadline replaces the idle deadline set by resetDeadline with @@ -303,11 +369,7 @@ func (w *IdleTimeoutWriter) clearDeadline() { return } - if err := w.Ctrl.SetWriteDeadline(w.Deadline.HardDeadline); err != nil { - w.unsupported = true - if c := w.Logger.Check(zapcore.DebugLevel, "could not clear write deadline"); c != nil { - c.Write(zap.Error(err)) - } + if !w.setWriteDeadline(w.Deadline.HardDeadline, "could not clear write deadline") { return } w.deadlineSet = false diff --git a/modules/caddyhttp/idletimeout_test.go b/modules/caddyhttp/idletimeout_test.go index 68025b6b9..666dfd89a 100644 --- a/modules/caddyhttp/idletimeout_test.go +++ b/modules/caddyhttp/idletimeout_test.go @@ -22,6 +22,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "sync" "testing" @@ -649,6 +650,7 @@ func TestIdleTimeoutWriter_ClearBetweenWrites(t *testing.T) { name string clear bool hard bool + idle time.Duration ops func(w *IdleTimeoutWriter) error expected []string }{ @@ -717,6 +719,23 @@ func TestIdleTimeoutWriter_ClearBetweenWrites(t *testing.T) { }, expected: []string{idleSet, zeroSet, idleSet, zeroSet}, }, + { + name: "hard deadline is restored for each operation", + clear: true, + hard: true, + idle: 3 * time.Hour, + ops: func(w *IdleTimeoutWriter) error { + if _, err := w.Write([]byte("x")); err != nil { + return err + } + if _, err := w.Write([]byte("x")); err != nil { + return err + } + w.HandlerDone() + return nil + }, + expected: []string{hardSet, hardSet, hardSet, hardSet, hardSet}, + }, { name: "handler done does nothing when not clearing", ops: func(w *IdleTimeoutWriter) error { @@ -727,11 +746,15 @@ func TestIdleTimeoutWriter_ClearBetweenWrites(t *testing.T) { expected: []string{idleSet}, }, } { + timeout := idle + if tc.idle > 0 { + timeout = tc.idle + } rec := &writeDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} w := &IdleTimeoutWriter{ ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: rec}, Ctrl: http.NewResponseController(rec), - Deadline: IdleDeadline{Timeout: idle}, + Deadline: IdleDeadline{Timeout: timeout}, Logger: zap.NewNop(), ClearBetweenWrites: tc.clear, } @@ -922,3 +945,225 @@ func TestIdleTimeoutReaderOverrideSuspendsMinRate(t *testing.T) { assert.True(t, deadlines[len(deadlines)-1].After(time.Now()), "the restored server-wide deadline should not have expired while overridden") } + +// TestIdleTimeoutReaderPausedHandlerHTTP2 checks that a handler that +// stops reading the body for longer than the idle timeout doesn't lose +// it on HTTP/2, where the read deadline is a timer that fails the body +// when it fires, even with no read in flight. The client isn't stalled: +// it's held back by flow control while the handler isn't reading. +func TestIdleTimeoutReaderPausedHandlerHTTP2(t *testing.T) { + const idle = 100 * time.Millisecond + const pause = 300 * time.Millisecond + const size = 4 << 20 // more than the HTTP/2 flow control window + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reader := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Start: time.Now(), Timeout: idle}, + Logger: zap.NewNop(), + ClearBetweenReads: r.ProtoMajor == 2, + } + defer reader.HandlerDone() + + if _, err := io.ReadFull(reader, make([]byte, 8)); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + time.Sleep(pause) + n, err := io.Copy(io.Discard, reader) + if err != nil { + http.Error(w, err.Error(), http.StatusRequestTimeout) + return + } + _, _ = fmt.Fprint(w, n+8) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + resp, err := srv.Client().Post(srv.URL, "application/octet-stream", bytes.NewReader(make([]byte, size))) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 2, resp.ProtoMajor) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode, string(body)) + assert.Equal(t, fmt.Sprint(size), string(body)) +} + +// TestIdleTimeoutReaderStalledClientHTTP2 checks that clearing the +// deadline between reads still aborts a read from a client that stops +// sending. +func TestIdleTimeoutReaderStalledClientHTTP2(t *testing.T) { + const idle = 100 * time.Millisecond + + readErr := make(chan error, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reader := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Start: time.Now(), Timeout: idle}, + Logger: zap.NewNop(), + ClearBetweenReads: r.ProtoMajor == 2, + } + defer reader.HandlerDone() + _, err := io.Copy(io.Discard, reader) + readErr <- err + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + pr, pw := io.Pipe() + defer pw.Close() + go func() { _, _ = pw.Write(make([]byte, 8)) }() // then stall + go func() { + resp, err := srv.Client().Post(srv.URL, "application/octet-stream", pr) + if err == nil { + resp.Body.Close() + } + }() + + select { + case err := <-readErr: + assert.ErrorIs(t, err, os.ErrDeadlineExceeded) + case <-time.After(5 * time.Second): + t.Fatal("read from a stalled client was not aborted") + } +} + +func TestIdleTimeoutReaderClearBetweenReads(t *testing.T) { + const idle = time.Hour + hard := time.Now().Add(2 * time.Hour) + + for i, tc := range []struct { + clear bool + hard bool + expected []string + }{ + // a read of "ab", then one returning io.EOF + {clear: false, expected: []string{"idle", "idle", "zero"}}, + {clear: true, expected: []string{"idle", "zero", "idle", "zero"}}, + {clear: true, hard: true, expected: []string{"idle", "hard", "idle", "zero"}}, + } { + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("ab")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: idle}, + Logger: zap.NewNop(), + ClearBetweenReads: tc.clear, + } + if tc.hard { + r.Deadline.HardDeadline = hard + } + + before := time.Now() + _, err := io.Copy(io.Discard, r) + after := time.Now() + require.NoError(t, err) + + var actual []string + for _, d := range w.snapshot() { + switch { + case d.IsZero(): + actual = append(actual, "zero") + case d.Equal(hard): + actual = append(actual, "hard") + case !d.Before(before.Add(idle)) && !d.After(after.Add(idle)): + actual = append(actual, "idle") + default: + actual = append(actual, "other") + } + } + assert.Equal(t, tc.expected, actual, "Test %d", i) + } +} + +// TestIdleTimeoutReaderHandlerDoneClearsReleasedHardDeadline checks +// that the hard deadline put back after a partial read doesn't outlive +// the handler when it returns without reading the rest of the body. +func TestIdleTimeoutReaderHandlerDoneClearsReleasedHardDeadline(t *testing.T) { + hard := time.Now().Add(time.Hour) + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("abcd")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Minute, HardDeadline: hard}, + Logger: zap.NewNop(), + ClearBetweenReads: true, + } + + _, err := r.Read(make([]byte, 2)) + require.NoError(t, err) + deadlines := w.snapshot() + require.Len(t, deadlines, 2) + assert.Equal(t, hard, deadlines[1], "the idle deadline should be released to the hard one") + + r.HandlerDone() + deadlines = w.snapshot() + require.Len(t, deadlines, 3) + assert.True(t, deadlines[2].IsZero(), "the released hard deadline should be cleared") +} + +// TestIdleTimeoutReaderRestoresHardDeadline checks that another handler's +// deadline update between reads cannot bypass the wrapper's hard ceiling. +func TestIdleTimeoutReaderRestoresHardDeadline(t *testing.T) { + for _, clear := range []bool{false, true} { + t.Run(fmt.Sprintf("clear=%t", clear), func(t *testing.T) { + hard := time.Now().Add(time.Minute) + rec := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + ctrl := http.NewResponseController(rec) + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("abcd")), + Ctrl: ctrl, + Deadline: IdleDeadline{Timeout: time.Hour, HardDeadline: hard}, + Logger: zap.NewNop(), + ClearBetweenReads: clear, + } + _, err := r.Read(make([]byte, 2)) + require.NoError(t, err) + // A handler can use its own response controller to change the deadline. + require.NoError(t, ctrl.SetReadDeadline(time.Time{})) + _, err = r.Read(make([]byte, 2)) + require.NoError(t, err) + deadlines := rec.snapshot() + assert.Equal(t, hard, deadlines[len(deadlines)-1]) + }) + } +} + +// TestIdleTimeoutWriterRestoresHardDeadline checks that another handler's +// deadline update cannot bypass the hard ceiling on writes or the final flush. +func TestIdleTimeoutWriterRestoresHardDeadline(t *testing.T) { + for _, op := range []string{"write", "read from", "flush", "handler done"} { + t.Run(op, func(t *testing.T) { + hard := time.Now().Add(time.Minute) + rec := &writeDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + ctrl := http.NewResponseController(rec) + w := &IdleTimeoutWriter{ + ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: rec}, + Ctrl: ctrl, + Deadline: IdleDeadline{Timeout: time.Hour, HardDeadline: hard}, + Logger: zap.NewNop(), + ClearBetweenWrites: true, + } + _, err := w.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, ctrl.SetWriteDeadline(time.Time{})) + switch op { + case "write": + _, err = w.Write([]byte("x")) + case "read from": + _, err = w.ReadFrom(strings.NewReader("x")) + case "flush": + err = w.FlushError() + case "handler done": + w.HandlerDone() + } + require.NoError(t, err) + assert.Equal(t, hard, rec.deadlines[len(rec.deadlines)-1]) + }) + } +} diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index 72c6679f4..ddfada9a3 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -597,8 +597,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { MinRate: s.ReadMinRate, HardDeadline: readHardDeadline, }, - Logger: s.logger, - DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + Logger: s.logger, + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + ClearBetweenReads: r.ProtoMajor == 2, } defer idleReader.HandlerDone() r.Body = idleReader diff --git a/modules/caddyhttp/timeouts/timeouts.go b/modules/caddyhttp/timeouts/timeouts.go index 90449d442..b3bdda4f5 100644 --- a/modules/caddyhttp/timeouts/timeouts.go +++ b/modules/caddyhttp/timeouts/timeouts.go @@ -114,8 +114,9 @@ func (t Timeouts) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht Timeout: t.ReadTimeout, MinRate: t.ReadMinRate, }, - Logger: t.logger, - DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + Logger: t.logger, + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + ClearBetweenReads: r.ProtoMajor == 2, } defer idleReader.HandlerDone() r.Body = idleReader