diff --git a/modules/caddyhttp/idletimeout.go b/modules/caddyhttp/idletimeout.go index 976580919..cd4fd4f70 100644 --- a/modules/caddyhttp/idletimeout.go +++ b/modules/caddyhttp/idletimeout.go @@ -122,13 +122,38 @@ type IdleTimeoutReader struct { // HTTP/2) for other protocols or requests without a body. DrainDeadline bool + // ClearBetweenReads makes the reader put back HardDeadline (no + // deadline unless a hard ceiling is configured) after every + // successful read, so the idle deadline only bounds reads in + // flight. It is required for HTTP/2, where the read deadline is a + // timer that fails the body when it fires, even with no read in + // flight: otherwise a handler that stops reading for longer than the + // timeout (e.g. while an upstream is slow to accept the body) loses + // the body, although the client is only held back by flow control. + // Other protocols only check the deadline during a read. + ClearBetweenReads bool + mu sync.Mutex unsupported bool - deadlineSet bool + armed armedDeadline finished bool terminalErr error } +// armedDeadline is which read deadline IdleTimeoutReader has armed. +type armedDeadline uint8 + +const ( + // deadlineNone means no deadline is armed. + deadlineNone armedDeadline = iota + // deadlineIdle means the idle deadline from IdleDeadline.next is + // armed. + deadlineIdle + // deadlineHard means only HardDeadline is armed, as put back + // between reads with ClearBetweenReads. + deadlineHard +) + func (r *IdleTimeoutReader) Read(p []byte) (int, error) { r.mu.Lock() if r.terminalErr != nil { @@ -150,6 +175,8 @@ func (r *IdleTimeoutReader) Read(p []byte) (int, error) { if !r.finished { r.clearDeadlineLocked() } + } else if r.ClearBetweenReads && !r.finished { + r.releaseDeadlineLocked() } r.mu.Unlock() @@ -158,8 +185,9 @@ func (r *IdleTimeoutReader) Read(p []byte) (int, error) { // HandlerDone prevents later body reads from using the response controller. // If the body is not finished and DrainDeadline is set, it leaves an idle -// deadline armed for net/http's post-handler drain. It must be called before -// the installing handler returns. +// deadline armed for net/http's post-handler drain. Otherwise, it clears a +// deadline left from a finished body or put back between reads. It must be +// called before the installing handler returns. func (r *IdleTimeoutReader) HandlerDone() { r.mu.Lock() defer r.mu.Unlock() @@ -167,9 +195,9 @@ func (r *IdleTimeoutReader) HandlerDone() { if r.finished { return } - if r.terminalErr != nil { + if r.terminalErr != nil || r.armed == deadlineHard { r.clearDeadlineLocked() - } else if r.DrainDeadline && !r.deadlineSet && !r.unsupported { + } else if r.DrainDeadline && r.armed != deadlineIdle && !r.unsupported { r.setDeadlineLocked("could not set final read deadline") } r.finished = true @@ -199,34 +227,61 @@ func (r *IdleTimeoutReader) Override(timeout time.Duration, minRate int64) (rest // rearmLocked moves an armed deadline to what the current settings give. func (r *IdleTimeoutReader) rearmLocked() { - if r.deadlineSet && !r.finished && !r.unsupported { + if r.armed == deadlineIdle && !r.finished && !r.unsupported { r.setDeadlineLocked("could not set read deadline") } } func (r *IdleTimeoutReader) setDeadlineLocked(logMessage string) { - if err := r.Ctrl.SetReadDeadline(r.Deadline.next()); err != nil { + if r.setReadDeadlineLocked(r.Deadline.next(), logMessage) { + r.armed = deadlineIdle + } +} + +// setReadDeadlineLocked always sets the requested deadline: another handler +// may have changed it through its own response controller since our last call. +// It reports whether the deadline was set successfully. +func (r *IdleTimeoutReader) setReadDeadlineLocked(deadline time.Time, logMessage string) bool { + if err := r.Ctrl.SetReadDeadline(deadline); err != nil { r.unsupported = true if c := r.Logger.Check(zapcore.DebugLevel, logMessage); c != nil { c.Write(zap.Error(err)) } + return false + } + return true +} + +// releaseDeadlineLocked replaces the idle deadline with HardDeadline. +func (r *IdleTimeoutReader) releaseDeadlineLocked() { + if r.armed != deadlineIdle || r.unsupported { return } - r.deadlineSet = true + + // once the hard deadline has passed, the deadline set before the + // read (capped to it) has already expired; keep it + if !r.Deadline.HardDeadline.IsZero() && !time.Now().Before(r.Deadline.HardDeadline) { + return + } + + if !r.setReadDeadlineLocked(r.Deadline.HardDeadline, "could not release read deadline") { + return + } + if r.Deadline.HardDeadline.IsZero() { + r.armed = deadlineNone + } else { + r.armed = deadlineHard + } } func (r *IdleTimeoutReader) clearDeadlineLocked() { - if !r.deadlineSet { + if r.armed == deadlineNone { return } - if err := r.Ctrl.SetReadDeadline(time.Time{}); err != nil { - r.unsupported = true - if c := r.Logger.Check(zapcore.DebugLevel, "could not clear read deadline"); c != nil { - c.Write(zap.Error(err)) - } + if !r.setReadDeadlineLocked(time.Time{}, "could not clear read deadline") { return } - r.deadlineSet = false + r.armed = deadlineNone } // IdleTimeoutWriter wraps a ResponseWriter with IdleDeadline, resetting @@ -270,6 +325,8 @@ type IdleTimeoutWriter struct { ClearBetweenWrites bool unsupported bool + // deadlineSet is whether the idle deadline is armed. clearDeadline + // leaves HardDeadline armed, if any, which nothing needs to undo. deadlineSet bool unflushed bool } @@ -279,14 +336,23 @@ func (w *IdleTimeoutWriter) resetDeadline() { return } - if err := w.Ctrl.SetWriteDeadline(w.Deadline.next()); err != nil { + if w.setWriteDeadline(w.Deadline.next(), "could not set write deadline") { + w.deadlineSet = true + } +} + +// setWriteDeadline always sets the requested deadline: another handler may +// have changed it through its own response controller since our last call. +// It reports whether the deadline was set successfully. +func (w *IdleTimeoutWriter) setWriteDeadline(deadline time.Time, logMessage string) bool { + if err := w.Ctrl.SetWriteDeadline(deadline); err != nil { w.unsupported = true - if c := w.Logger.Check(zapcore.DebugLevel, "could not set write deadline"); c != nil { + if c := w.Logger.Check(zapcore.DebugLevel, logMessage); c != nil { c.Write(zap.Error(err)) } - return + return false } - w.deadlineSet = true + return true } // clearDeadline replaces the idle deadline set by resetDeadline with @@ -303,11 +369,7 @@ func (w *IdleTimeoutWriter) clearDeadline() { return } - if err := w.Ctrl.SetWriteDeadline(w.Deadline.HardDeadline); err != nil { - w.unsupported = true - if c := w.Logger.Check(zapcore.DebugLevel, "could not clear write deadline"); c != nil { - c.Write(zap.Error(err)) - } + if !w.setWriteDeadline(w.Deadline.HardDeadline, "could not clear write deadline") { return } w.deadlineSet = false diff --git a/modules/caddyhttp/idletimeout_test.go b/modules/caddyhttp/idletimeout_test.go index 68025b6b9..666dfd89a 100644 --- a/modules/caddyhttp/idletimeout_test.go +++ b/modules/caddyhttp/idletimeout_test.go @@ -22,6 +22,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "sync" "testing" @@ -649,6 +650,7 @@ func TestIdleTimeoutWriter_ClearBetweenWrites(t *testing.T) { name string clear bool hard bool + idle time.Duration ops func(w *IdleTimeoutWriter) error expected []string }{ @@ -717,6 +719,23 @@ func TestIdleTimeoutWriter_ClearBetweenWrites(t *testing.T) { }, expected: []string{idleSet, zeroSet, idleSet, zeroSet}, }, + { + name: "hard deadline is restored for each operation", + clear: true, + hard: true, + idle: 3 * time.Hour, + ops: func(w *IdleTimeoutWriter) error { + if _, err := w.Write([]byte("x")); err != nil { + return err + } + if _, err := w.Write([]byte("x")); err != nil { + return err + } + w.HandlerDone() + return nil + }, + expected: []string{hardSet, hardSet, hardSet, hardSet, hardSet}, + }, { name: "handler done does nothing when not clearing", ops: func(w *IdleTimeoutWriter) error { @@ -727,11 +746,15 @@ func TestIdleTimeoutWriter_ClearBetweenWrites(t *testing.T) { expected: []string{idleSet}, }, } { + timeout := idle + if tc.idle > 0 { + timeout = tc.idle + } rec := &writeDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} w := &IdleTimeoutWriter{ ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: rec}, Ctrl: http.NewResponseController(rec), - Deadline: IdleDeadline{Timeout: idle}, + Deadline: IdleDeadline{Timeout: timeout}, Logger: zap.NewNop(), ClearBetweenWrites: tc.clear, } @@ -922,3 +945,225 @@ func TestIdleTimeoutReaderOverrideSuspendsMinRate(t *testing.T) { assert.True(t, deadlines[len(deadlines)-1].After(time.Now()), "the restored server-wide deadline should not have expired while overridden") } + +// TestIdleTimeoutReaderPausedHandlerHTTP2 checks that a handler that +// stops reading the body for longer than the idle timeout doesn't lose +// it on HTTP/2, where the read deadline is a timer that fails the body +// when it fires, even with no read in flight. The client isn't stalled: +// it's held back by flow control while the handler isn't reading. +func TestIdleTimeoutReaderPausedHandlerHTTP2(t *testing.T) { + const idle = 100 * time.Millisecond + const pause = 300 * time.Millisecond + const size = 4 << 20 // more than the HTTP/2 flow control window + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reader := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Start: time.Now(), Timeout: idle}, + Logger: zap.NewNop(), + ClearBetweenReads: r.ProtoMajor == 2, + } + defer reader.HandlerDone() + + if _, err := io.ReadFull(reader, make([]byte, 8)); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + time.Sleep(pause) + n, err := io.Copy(io.Discard, reader) + if err != nil { + http.Error(w, err.Error(), http.StatusRequestTimeout) + return + } + _, _ = fmt.Fprint(w, n+8) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + resp, err := srv.Client().Post(srv.URL, "application/octet-stream", bytes.NewReader(make([]byte, size))) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 2, resp.ProtoMajor) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode, string(body)) + assert.Equal(t, fmt.Sprint(size), string(body)) +} + +// TestIdleTimeoutReaderStalledClientHTTP2 checks that clearing the +// deadline between reads still aborts a read from a client that stops +// sending. +func TestIdleTimeoutReaderStalledClientHTTP2(t *testing.T) { + const idle = 100 * time.Millisecond + + readErr := make(chan error, 1) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reader := &IdleTimeoutReader{ + ReadCloser: r.Body, + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Start: time.Now(), Timeout: idle}, + Logger: zap.NewNop(), + ClearBetweenReads: r.ProtoMajor == 2, + } + defer reader.HandlerDone() + _, err := io.Copy(io.Discard, reader) + readErr <- err + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + pr, pw := io.Pipe() + defer pw.Close() + go func() { _, _ = pw.Write(make([]byte, 8)) }() // then stall + go func() { + resp, err := srv.Client().Post(srv.URL, "application/octet-stream", pr) + if err == nil { + resp.Body.Close() + } + }() + + select { + case err := <-readErr: + assert.ErrorIs(t, err, os.ErrDeadlineExceeded) + case <-time.After(5 * time.Second): + t.Fatal("read from a stalled client was not aborted") + } +} + +func TestIdleTimeoutReaderClearBetweenReads(t *testing.T) { + const idle = time.Hour + hard := time.Now().Add(2 * time.Hour) + + for i, tc := range []struct { + clear bool + hard bool + expected []string + }{ + // a read of "ab", then one returning io.EOF + {clear: false, expected: []string{"idle", "idle", "zero"}}, + {clear: true, expected: []string{"idle", "zero", "idle", "zero"}}, + {clear: true, hard: true, expected: []string{"idle", "hard", "idle", "zero"}}, + } { + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("ab")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: idle}, + Logger: zap.NewNop(), + ClearBetweenReads: tc.clear, + } + if tc.hard { + r.Deadline.HardDeadline = hard + } + + before := time.Now() + _, err := io.Copy(io.Discard, r) + after := time.Now() + require.NoError(t, err) + + var actual []string + for _, d := range w.snapshot() { + switch { + case d.IsZero(): + actual = append(actual, "zero") + case d.Equal(hard): + actual = append(actual, "hard") + case !d.Before(before.Add(idle)) && !d.After(after.Add(idle)): + actual = append(actual, "idle") + default: + actual = append(actual, "other") + } + } + assert.Equal(t, tc.expected, actual, "Test %d", i) + } +} + +// TestIdleTimeoutReaderHandlerDoneClearsReleasedHardDeadline checks +// that the hard deadline put back after a partial read doesn't outlive +// the handler when it returns without reading the rest of the body. +func TestIdleTimeoutReaderHandlerDoneClearsReleasedHardDeadline(t *testing.T) { + hard := time.Now().Add(time.Hour) + w := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("abcd")), + Ctrl: http.NewResponseController(w), + Deadline: IdleDeadline{Timeout: time.Minute, HardDeadline: hard}, + Logger: zap.NewNop(), + ClearBetweenReads: true, + } + + _, err := r.Read(make([]byte, 2)) + require.NoError(t, err) + deadlines := w.snapshot() + require.Len(t, deadlines, 2) + assert.Equal(t, hard, deadlines[1], "the idle deadline should be released to the hard one") + + r.HandlerDone() + deadlines = w.snapshot() + require.Len(t, deadlines, 3) + assert.True(t, deadlines[2].IsZero(), "the released hard deadline should be cleared") +} + +// TestIdleTimeoutReaderRestoresHardDeadline checks that another handler's +// deadline update between reads cannot bypass the wrapper's hard ceiling. +func TestIdleTimeoutReaderRestoresHardDeadline(t *testing.T) { + for _, clear := range []bool{false, true} { + t.Run(fmt.Sprintf("clear=%t", clear), func(t *testing.T) { + hard := time.Now().Add(time.Minute) + rec := &readDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + ctrl := http.NewResponseController(rec) + r := &IdleTimeoutReader{ + ReadCloser: io.NopCloser(strings.NewReader("abcd")), + Ctrl: ctrl, + Deadline: IdleDeadline{Timeout: time.Hour, HardDeadline: hard}, + Logger: zap.NewNop(), + ClearBetweenReads: clear, + } + _, err := r.Read(make([]byte, 2)) + require.NoError(t, err) + // A handler can use its own response controller to change the deadline. + require.NoError(t, ctrl.SetReadDeadline(time.Time{})) + _, err = r.Read(make([]byte, 2)) + require.NoError(t, err) + deadlines := rec.snapshot() + assert.Equal(t, hard, deadlines[len(deadlines)-1]) + }) + } +} + +// TestIdleTimeoutWriterRestoresHardDeadline checks that another handler's +// deadline update cannot bypass the hard ceiling on writes or the final flush. +func TestIdleTimeoutWriterRestoresHardDeadline(t *testing.T) { + for _, op := range []string{"write", "read from", "flush", "handler done"} { + t.Run(op, func(t *testing.T) { + hard := time.Now().Add(time.Minute) + rec := &writeDeadlineRecorder{ResponseRecorder: httptest.NewRecorder()} + ctrl := http.NewResponseController(rec) + w := &IdleTimeoutWriter{ + ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: rec}, + Ctrl: ctrl, + Deadline: IdleDeadline{Timeout: time.Hour, HardDeadline: hard}, + Logger: zap.NewNop(), + ClearBetweenWrites: true, + } + _, err := w.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, ctrl.SetWriteDeadline(time.Time{})) + switch op { + case "write": + _, err = w.Write([]byte("x")) + case "read from": + _, err = w.ReadFrom(strings.NewReader("x")) + case "flush": + err = w.FlushError() + case "handler done": + w.HandlerDone() + } + require.NoError(t, err) + assert.Equal(t, hard, rec.deadlines[len(rec.deadlines)-1]) + }) + } +} diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index 72c6679f4..ddfada9a3 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -597,8 +597,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { MinRate: s.ReadMinRate, HardDeadline: readHardDeadline, }, - Logger: s.logger, - DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + Logger: s.logger, + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + ClearBetweenReads: r.ProtoMajor == 2, } defer idleReader.HandlerDone() r.Body = idleReader diff --git a/modules/caddyhttp/timeouts/timeouts.go b/modules/caddyhttp/timeouts/timeouts.go index 90449d442..b3bdda4f5 100644 --- a/modules/caddyhttp/timeouts/timeouts.go +++ b/modules/caddyhttp/timeouts/timeouts.go @@ -114,8 +114,9 @@ func (t Timeouts) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht Timeout: t.ReadTimeout, MinRate: t.ReadMinRate, }, - Logger: t.logger, - DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + Logger: t.logger, + DrainDeadline: r.ProtoMajor == 1 && r.ContentLength != 0, + ClearBetweenReads: r.ProtoMajor == 2, } defer idleReader.HandlerDone() r.Body = idleReader