From e0c680108b9bb384b188622af16773006494ff4a Mon Sep 17 00:00:00 2001 From: jotikrishna <213903218+jotikrishna@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:57:55 +0530 Subject: [PATCH 1/2] caddyhttp: don't flush a hijacked response Since #7945, Server.ServeHTTP flushes the response in a deferred func when an HTTP/1.x request that is being access-logged took at least write_timeout. The deferred func already skips http.ErrHijacked, on the assumption that Flush reports a hijacked response that way. It does not. (*http.response).Hijack releases the bufio.Writer, and (*http.response).FlushError calls w.w.Flush() without checking for the hijack. Only Write and WriteHeader consult hijacked() and return ErrHijacked. So when the handler hijacked the connection, as reverse_proxy does when it passes through a 101 Switching Protocols upgrade, and the upgraded stream outlives write_timeout, the deferred flush reaches a nil bufio.Writer and panics with an invalid memory address. net/http recovers it and it fires only after the upgraded stream ended, so clients never notice, but every such connection ends with an error-level stack trace in the logs. responseRecorder already tracks the hijack for DetachAfterHijack and WriteResponse, so report it from FlushError too, which is what the deferred func expects. Fixes #8151 --- modules/caddyhttp/responsewriter.go | 7 + .../caddyhttp/responsewriter_hijack_test.go | 226 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 modules/caddyhttp/responsewriter_hijack_test.go diff --git a/modules/caddyhttp/responsewriter.go b/modules/caddyhttp/responsewriter.go index 68f9b2f65..e95fc63ad 100644 --- a/modules/caddyhttp/responsewriter.go +++ b/modules/caddyhttp/responsewriter.go @@ -255,6 +255,13 @@ func (rr *responseRecorder) WriteResponse() error { // FlushError will suppress actual flushing if the response is buffered. See: // https://github.com/caddyserver/caddy/issues/6144 func (rr *responseRecorder) FlushError() error { + // Hijack releases the underlying bufio.Writer, and net/http's FlushError + // dereferences it without checking for the hijack, so flushing after an + // upgrade panics with a nil pointer dereference. Write and WriteHeader + // already report the hijack; FlushError has to do the same. See #8151. + if rr.hijacked { + return http.ErrHijacked + } if rr.stream { //nolint:bodyclose return http.NewResponseController(rr.ResponseWriterWrapper).Flush() diff --git a/modules/caddyhttp/responsewriter_hijack_test.go b/modules/caddyhttp/responsewriter_hijack_test.go new file mode 100644 index 000000000..d00b6d02a --- /dev/null +++ b/modules/caddyhttp/responsewriter_hijack_test.go @@ -0,0 +1,226 @@ +package caddyhttp + +import ( + "bufio" + "errors" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/caddyserver/caddy/v2" + "go.uber.org/zap" +) + +// hijackableRecorder is an http.ResponseWriter that implements http.Hijacker the +// way net/http does. This matters: after a successful Hijack the real +// (*http.response) has released its bufio.Writer (w.w = nil) and requires +// Write/WriteHeader to return ErrHijacked. FlushError, however, dereferences +// that writer without checking for the hijack, so flushing afterwards is not +// merely wrong, it is a nil pointer dereference. +type hijackableRecorder struct { + header http.Header + + hijacked bool + flushed int +} + +func newHijackableRecorder() *hijackableRecorder { + return &hijackableRecorder{header: make(http.Header)} +} + +func (h *hijackableRecorder) Header() http.Header { return h.header } + +func (h *hijackableRecorder) WriteHeader(int) { + if h.hijacked { + return + } +} + +func (h *hijackableRecorder) Write(p []byte) (int, error) { + if h.hijacked { + return 0, http.ErrHijacked + } + return len(p), nil +} + +// Flush mirrors stdlib (*http.response).FlushError: it does not consult the +// hijack state, which is exactly how the regression in #8151 becomes a panic. +func (h *hijackableRecorder) Flush() { + if h.hijacked { + panic("runtime error: invalid memory address or nil pointer dereference") + } + h.flushed++ +} + +func (h *hijackableRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { + h.hijacked = true + c1, c2 := net.Pipe() + go func() { _ = c2.Close(); _ = c1.Close() }() + return c1, bufio.NewReadWriter(bufio.NewReader(c1), bufio.NewWriter(c1)), nil +} + +// flushAfterHijack models the deferred flush in Server.ServeHTTP. Server side it +// reads: +// +// flushErr := http.NewResponseController(wrec).Flush() +// if flushErr != nil && +// !errors.Is(flushErr, http.ErrHijacked) && +// !errors.Is(flushErr, http.ErrNotSupported) { +// writeErr = flushErr +// } +// +// so it already expects ErrHijacked from a hijacked connection. +func flushAfterHijack(t *testing.T, w http.ResponseWriter) { + t.Helper() + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("flushing a hijacked response panicked: %v", rec) + } + }() + err := http.NewResponseController(w).Flush() + if err != nil && !errors.Is(err, http.ErrHijacked) { + t.Fatalf("expected http.ErrHijacked or nil from a hijacked response, got %v", err) + } +} + +// TestResponseRecorderFlushAfterHijack is the regression test for +// caddyserver/caddy#8151. +// +// Since #7945, Server.ServeHTTP defers a flush so the access log can observe a +// write error. When the handler hijacked the connection (reverse_proxy passing +// through a 101 Switching Protocols upgrade) and the upgraded stream outlives +// write_timeout, that deferred flush runs after the hijack, reaches net/http and +// panics with a nil pointer dereference. +// +// The recorder must remember the hijack and report it from FlushError. +func TestResponseRecorderFlushAfterHijack(t *testing.T) { + underlying := newHijackableRecorder() + + // nil shouldBuffer means the recorder streams straight to the underlying + // writer, the mode an upgraded reverse_proxy response uses. + rec := NewResponseRecorder(underlying, nil, nil).(*responseRecorder) + + // reverse_proxy writes the 101 Switching Protocols status before hijacking; + // WriteHeader is what puts the recorder into streaming mode (rr.stream), and + // streaming mode is the branch of FlushError that reaches net/http. + rec.WriteHeader(http.StatusSwitchingProtocols) + + if _, _, err := rec.Hijack(); err != nil { + t.Fatalf("Hijack() returned unexpected error: %v", err) + } + + flushAfterHijack(t, rec) + + if underlying.flushed != 0 { + t.Fatalf("underlying writer was flushed %d time(s) after the hijack; "+ + "a hijacked response must not be flushed", underlying.flushed) + } +} + +// TestResponseRecorderFlushWithoutHijack guards the control case: the fix must +// not turn FlushError into a no-op for ordinary streamed responses. +func TestResponseRecorderFlushWithoutHijack(t *testing.T) { + underlying := newHijackableRecorder() + rec := NewResponseRecorder(underlying, nil, nil).(*responseRecorder) + + // Streaming mode is entered via WriteHeader, as it is for a real response. + rec.WriteHeader(http.StatusOK) + + if err := http.NewResponseController(rec).Flush(); err != nil { + t.Fatalf("unexpected error flushing a non-hijacked streamed response: %v", err) + } + if underlying.flushed != 1 { + t.Fatalf("expected exactly 1 flush on the underlying writer, got %d", underlying.flushed) + } +} + +// TestResponseRecorderBufferedFlushUnaffected guards the pre-existing #6144 +// behaviour: buffered responses suppress the flush regardless of hijack state. +func TestResponseRecorderBufferedFlushUnaffected(t *testing.T) { + underlying := newHijackableRecorder() + alwaysBuffer := func(int, http.Header) bool { return true } + rec := NewResponseRecorder(underlying, nil, alwaysBuffer).(*responseRecorder) + + rec.WriteHeader(http.StatusOK) + if !rec.Buffered() { + t.Fatal("expected a buffered response") + } + if err := http.NewResponseController(rec).Flush(); err != nil { + t.Fatalf("unexpected error flushing a buffered response: %v", err) + } + if underlying.flushed != 0 { + t.Fatalf("a buffered response should suppress the flush, got %d flush(es)", underlying.flushed) + } +} + +// TestServerServeHTTPDeferredFlushAfterHijack drives the real Server.ServeHTTP +// path end to end, which is where the panic was observed: an upgraded connection +// that outlives write_timeout is flushed from the deferred func after the +// handler hijacked the connection. +func TestServerServeHTTPDeferredFlushAfterHijack(t *testing.T) { + underlying := newHijackableRecorder() + + const handlerDuration = 5 * time.Millisecond + + handler := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { + // reverse_proxy passes a 101 through, then hijacks. Writing the status + // first is what selects the recorder's streaming branch. + w.WriteHeader(http.StatusSwitchingProtocols) + + hj, ok := w.(http.Hijacker) + if !ok { + t.Error("recorder does not implement http.Hijacker") + return nil + } + conn, _, err := hj.Hijack() + if err != nil { + t.Errorf("hijack failed: %v", err) + return nil + } + defer conn.Close() + // The upgrade outlives write_timeout, which is what makes the deferred + // flush run at all. + time.Sleep(handlerDuration) + return nil + }) + + srv := &Server{ + primaryHandlerChain: handler, + WriteTimeout: caddy.Duration(time.Millisecond), + Logs: &ServerLogConfig{}, + } + // Field names are unexported; set them directly as the tests in this package do. + srv.logger = zap.NewNop() + srv.errorLogger = zap.NewNop() + srv.accessLogger = zap.NewNop() + + r := httptest.NewRequest(http.MethodGet, "http://localhost/", nil) + r.ProtoMajor, r.ProtoMinor = 1, 1 + + // Record the panic so the failure names the regression rather than the test + // binary crashing. + panicked := make(chan any, 1) + func() { + defer func() { + if rec := recover(); rec != nil { + panicked <- rec + } + }() + srv.ServeHTTP(underlying, r) + }() + + select { + case rec := <-panicked: + t.Fatalf("ServeHTTP panicked after an upgraded connection outlived write_timeout: %v", rec) + default: + } + + if !underlying.hijacked { + t.Fatal("handler did not hijack the connection; test did not exercise the upgrade path") + } + if underlying.flushed != 0 { + t.Fatalf("deferred flush reached the hijacked writer %d time(s)", underlying.flushed) + } +} From 0d0c2d3cd58674e061aaca19a71415d94083a249 Mon Sep 17 00:00:00 2001 From: jotikrishna <213903218+jotikrishna@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:48:33 +0530 Subject: [PATCH 2/2] caddyhttp: clarify the hijack comment and mirror FlushError in the test Address review feedback from WeidiDeng on #8164: - The comment claimed Write and WriteHeader already report the hijack, but WriteHeader does not return anything: after a hijack it is a silent no-op. Only Write returns ErrHijacked. Reword to say what each actually does. - The test fake implemented Flush; net/http responses implement FlushError, which is what http.NewResponseController prefers, so implement FlushError instead to mirror the real interface shape. --- modules/caddyhttp/responsewriter.go | 9 +++++---- modules/caddyhttp/responsewriter_hijack_test.go | 17 ++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/modules/caddyhttp/responsewriter.go b/modules/caddyhttp/responsewriter.go index e95fc63ad..e5de84a48 100644 --- a/modules/caddyhttp/responsewriter.go +++ b/modules/caddyhttp/responsewriter.go @@ -255,10 +255,11 @@ func (rr *responseRecorder) WriteResponse() error { // FlushError will suppress actual flushing if the response is buffered. See: // https://github.com/caddyserver/caddy/issues/6144 func (rr *responseRecorder) FlushError() error { - // Hijack releases the underlying bufio.Writer, and net/http's FlushError - // dereferences it without checking for the hijack, so flushing after an - // upgrade panics with a nil pointer dereference. Write and WriteHeader - // already report the hijack; FlushError has to do the same. See #8151. + // Hijack releases the underlying bufio.Writer. net/http's Write returns + // ErrHijacked from then on and WriteHeader silently does nothing, but + // net/http's FlushError does neither and dereferences the released + // writer, so flushing after an upgrade panics with a nil pointer + // dereference. Report the hijack from here too. See #8151. if rr.hijacked { return http.ErrHijacked } diff --git a/modules/caddyhttp/responsewriter_hijack_test.go b/modules/caddyhttp/responsewriter_hijack_test.go index d00b6d02a..32346c770 100644 --- a/modules/caddyhttp/responsewriter_hijack_test.go +++ b/modules/caddyhttp/responsewriter_hijack_test.go @@ -15,10 +15,11 @@ import ( // hijackableRecorder is an http.ResponseWriter that implements http.Hijacker the // way net/http does. This matters: after a successful Hijack the real -// (*http.response) has released its bufio.Writer (w.w = nil) and requires -// Write/WriteHeader to return ErrHijacked. FlushError, however, dereferences -// that writer without checking for the hijack, so flushing afterwards is not -// merely wrong, it is a nil pointer dereference. +// (*http.response) has released its bufio.Writer (w.w = nil). Write returns +// ErrHijacked from then on and WriteHeader silently does nothing, but +// (*http.response).FlushError does neither and dereferences the released +// writer, so flushing afterwards is not merely wrong, it is a nil pointer +// dereference. type hijackableRecorder struct { header http.Header @@ -45,13 +46,15 @@ func (h *hijackableRecorder) Write(p []byte) (int, error) { return len(p), nil } -// Flush mirrors stdlib (*http.response).FlushError: it does not consult the -// hijack state, which is exactly how the regression in #8151 becomes a panic. -func (h *hijackableRecorder) Flush() { +// FlushError mirrors stdlib (*http.response).FlushError: it does not consult +// the hijack state and dereferences the released writer, which is exactly how +// the regression in #8151 becomes a panic. +func (h *hijackableRecorder) FlushError() error { if h.hijacked { panic("runtime error: invalid memory address or nil pointer dereference") } h.flushed++ + return nil } func (h *hijackableRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {