caddy/modules/caddyhttp/reverseproxy/active_health_test.go
Jae Birdsall 098cc69fe4
Some checks are pending
Tests / test (./cmd/caddy/caddy, ~1.27.0, macos-latest, 0, 1.27, mac) (push) Waiting to run
Tests / test (./cmd/caddy/caddy, ~1.27.0, ubuntu-latest, 0, 1.27, linux) (push) Waiting to run
Tests / test (./cmd/caddy/caddy.exe, ~1.27.0, windows-latest, True, 1.27, windows) (push) Waiting to run
Tests / test (s390x on IBM Z) (push) Waiting to run
Tests / goreleaser-check (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, aix) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, darwin) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, dragonfly) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, freebsd) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, illumos) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, linux) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, netbsd) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, openbsd) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, solaris) (push) Waiting to run
Cross-Build / build (~1.26.0, 1.26, windows) (push) Waiting to run
Lint / lint (push) Waiting to run
Lint / lint-1 (push) Waiting to run
Lint / lint-2 (push) Waiting to run
Lint / govulncheck (push) Waiting to run
Lint / dependency-review (push) Waiting to run
OpenSSF Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
reverseproxy: pick the health check scheme by the port it connects to (#8395)
With health_upstream, the active health check connects to another
address, but OverrideHealthCheckScheme was given the upstream's port.
An upstream on a TLS port with health_upstream on a port listed in
tls_except_ports was checked over HTTPS against a plain HTTP port; every
check failed and the upstream was marked down.
2026-11-14 01:25:05 -05:00

277 lines
9.3 KiB
Go

// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package reverseproxy
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"go.uber.org/zap"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyevents"
)
// newActiveHandler builds a minimal Handler with active health checks
// configured against addr, provisions its single upstream, and returns
// the handler, its upstream, and a cancel func the caller must defer.
func newActiveHandler(t *testing.T, addr, uri string, fails int) (*Handler, *Upstream, context.CancelFunc) {
t.Helper()
caddyCtx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
eventsApp := new(caddyevents.App)
if err := eventsApp.Provision(caddyCtx); err != nil {
t.Fatalf("provisioning events app: %v", err)
}
u := &Upstream{Dial: addr}
h := &Handler{
ctx: caddyCtx,
logger: zap.NewNop(),
events: eventsApp,
Upstreams: UpstreamPool{u},
HealthChecks: &HealthChecks{
Active: &ActiveHealthChecks{
URI: uri,
Fails: fails,
},
},
}
h.provisionUpstream(u, false)
if err := h.HealthChecks.Active.Provision(caddyCtx, h); err != nil {
t.Fatalf("provisioning active health checks: %v", err)
}
return h, u, cancel
}
// runActiveHealthCheck synchronously performs one active health check round
// for the handler's upstream (what doActiveHealthCheckForAllHosts does per
// tick, minus the goroutine).
func runActiveHealthCheck(t *testing.T, h *Handler, u *Upstream) {
t.Helper()
dialInfo, err := u.fillDialInfo(caddy.NewReplacer())
if err != nil {
t.Fatalf("filling dial info: %v", err)
}
if err := h.doActiveHealthCheck(dialInfo, dialInfo.Address, u.Dial, u); err != nil {
t.Fatalf("active health check: %v", err)
}
}
// drainHostsPool removes every entry from the global static host pool,
// deleting each exactly as many times as it was stored so the pool is
// empty for subsequent tests.
func drainHostsPool() {
var keys []any
hosts.Range(func(key, _ any) bool {
keys = append(keys, key)
return true
})
for _, key := range keys {
if refs, ok := hosts.References(key); ok {
for range refs {
_, _ = hosts.Delete(key)
}
}
}
}
// TestActiveHealthChecksSameAddressDifferentChecksAreIndependent is a
// regression test for https://github.com/caddyserver/caddy/issues/7870:
// two handlers that dial the same upstream address but run different
// active health checks (different health_uri) must keep independent
// health state. Before the fix, the consecutive pass/fail counters lived
// on the Host, which is shared by dial address, so both checkers mutated
// the same counters: one vhost's failing probes could push the other
// vhost's upstream over its own fails threshold and knock it out of that
// vhost's data path.
func TestActiveHealthChecksSameAddressDifferentChecksAreIndependent(t *testing.T) {
resetDynamicHosts()
defer drainHostsPool()
// one backend node serving two vhosts: vhost A's health endpoint is
// down, vhost B's is up (except for one transient failure below)
var vhostBUp atomic.Bool
vhostBUp.Store(true)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/vhost-a/health":
w.WriteHeader(http.StatusServiceUnavailable)
case "/vhost-b/health":
if vhostBUp.Load() {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusServiceUnavailable)
}
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer srv.Close()
addr := strings.TrimPrefix(srv.URL, "http://")
hA, uA, cancelA := newActiveHandler(t, addr, "/vhost-a/health", 5)
defer cancelA()
hB, uB, cancelB := newActiveHandler(t, addr, "/vhost-b/health", 3)
defer cancelB()
// vhost A's checker observes two consecutive failures — below its own
// fails=5 threshold, so these must affect no one's health status
runActiveHealthCheck(t, hA, uA)
runActiveHealthCheck(t, hA, uA)
// vhost B's checker observes a single transient failure; B tolerates
// up to fails=3 consecutive failures, so it must remain healthy
vhostBUp.Store(false)
runActiveHealthCheck(t, hB, uB)
if !uA.Healthy() {
t.Errorf("vhost A's upstream should still be healthy after 2 of 5 tolerated failures")
}
if !uB.Healthy() {
t.Errorf("vhost B's upstream was marked unhealthy after a single failed probe (fails=3); " +
"its health state was polluted by vhost A's health check against the same address")
}
}
// TestActiveHealthCheckBodyKeepsUnknownPlaceholders is a regression test for
// https://github.com/caddyserver/caddy/issues/7021: an active health check
// body is user-supplied and is commonly JSON, so its braces must not be read
// as placeholders and blanked. Only globals are available to this replacer, so
// replacing everything sent an empty body for any JSON health_request_body.
func TestActiveHealthCheckBodyKeepsUnknownPlaceholders(t *testing.T) {
resetDynamicHosts()
defer drainHostsPool()
const jsonBody = `{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`
var gotBody atomic.Value
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Errorf("reading request body: %v", err)
}
gotBody.Store(string(body))
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
addr := strings.TrimPrefix(srv.URL, "http://")
h, u, cancel := newActiveHandler(t, addr, "/health", 1)
defer cancel()
h.HealthChecks.Active.Body = jsonBody
runActiveHealthCheck(t, h, u)
if got, want := gotBody.Load(), jsonBody; got != want {
t.Errorf("health check body = %q, want %q", got, want)
}
}
// TestActiveHealthCheckBodyStillReplacesKnownPlaceholders guards the other
// direction: switching to ReplaceKnown must not stop real placeholders from
// being replaced.
func TestActiveHealthCheckBodyStillReplacesKnownPlaceholders(t *testing.T) {
resetDynamicHosts()
defer drainHostsPool()
t.Setenv("CADDY_TEST_HEALTH_TOKEN", "s3cret")
var gotBody atomic.Value
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Errorf("reading request body: %v", err)
}
gotBody.Store(string(body))
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
addr := strings.TrimPrefix(srv.URL, "http://")
h, u, cancel := newActiveHandler(t, addr, "/health", 1)
defer cancel()
h.HealthChecks.Active.Body = `{"token":"{env.CADDY_TEST_HEALTH_TOKEN}"}`
runActiveHealthCheck(t, h, u)
if got, want := gotBody.Load(), `{"token":"s3cret"}`; got != want {
t.Errorf("health check body = %q, want %q", got, want)
}
}
// schemeRecordingTransport keeps HTTPTransport's health check scheme
// override but answers every request itself, recording the URL used.
type schemeRecordingTransport struct {
HTTPTransport
got atomic.Value // string
}
func (rt *schemeRecordingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
rt.got.Store(req.URL.Scheme + "://" + req.URL.Host)
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil
}
// TestActiveHealthCheckSchemeUsesHealthCheckPort checks that tls_except_ports
// is applied to the port the active health check connects to, so a
// health_upstream or health_port on an excepted port is checked over plain
// HTTP even though the upstream itself uses TLS.
func TestActiveHealthCheckSchemeUsesHealthCheckPort(t *testing.T) {
for _, tc := range []struct {
name string
healthUpstream string
healthPort int
want string
}{
{name: "upstream port", want: "https://127.0.0.1:8443"},
{name: "health_upstream on excepted port", healthUpstream: "127.0.0.1:8080", want: "http://127.0.0.1:8080"},
{name: "health_upstream on TLS port", healthUpstream: "127.0.0.1:9443", want: "https://127.0.0.1:9443"},
{name: "health_port on excepted port", healthPort: 8080, want: "http://127.0.0.1:8080"},
} {
t.Run(tc.name, func(t *testing.T) {
h, u, cancel := newActiveHandler(t, "127.0.0.1:8443", "/health", 1)
defer cancel()
defer drainHostsPool()
rt := &schemeRecordingTransport{HTTPTransport: HTTPTransport{TLS: &TLSConfig{ExceptPorts: []string{"8080"}}}}
h.Transport = rt
h.HealthChecks.Active.httpClient.Transport = rt
h.HealthChecks.Active.Upstream = tc.healthUpstream
h.HealthChecks.Active.Port = tc.healthPort
hostAddr := u.Dial
if tc.healthPort != 0 {
hostAddr = "127.0.0.1:8080" // what doActiveHealthCheckForAllHosts passes
}
dialInfo, err := u.fillDialInfo(caddy.NewReplacer())
if err != nil {
t.Fatal(err)
}
if err := h.doActiveHealthCheck(dialInfo, hostAddr, u.Dial, u); err != nil {
t.Fatalf("active health check: %v", err)
}
if got, _ := rt.got.Load().(string); got != tc.want {
t.Errorf("health check URL = %q, want %q", got, tc.want)
}
})
}
}