diff --git a/modules/caddyhttp/celmatcher.go b/modules/caddyhttp/celmatcher.go index 7ceab02a8..0118cc7c8 100644 --- a/modules/caddyhttp/celmatcher.go +++ b/modules/caddyhttp/celmatcher.go @@ -22,6 +22,7 @@ import ( "net/http" "reflect" "regexp" + "slices" "strings" "time" @@ -70,6 +71,7 @@ type MatchExpression struct { Name string `json:"name,omitempty"` expandedExpr string + placeholders []string prg cel.Program ta types.Adapter @@ -192,6 +194,8 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error { return fmt.Errorf("CEL request matcher expects return type of bool, not %s", checked.OutputType()) } + m.placeholders = celPlaceholderNames(checked.NativeRep().Expr()) + // compile the "program" m.prg, err = env.Program(checked, cel.EvalOptions(cel.OptOptimize)) if err != nil { @@ -200,6 +204,13 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error { return nil } +// Placeholders returns the names of the placeholders the provisioned +// expression reads. A name that is only known at evaluation time is +// returned as an empty string. +func (m MatchExpression) Placeholders() []string { + return slices.Clone(m.placeholders) +} + // Match returns true if r matches m. func (m MatchExpression) Match(r *http.Request) bool { match, err := m.MatchWithError(r) @@ -729,6 +740,23 @@ func CELValueToMapStrList(data ref.Val) (map[string][]string, error) { return mapStrListStr, nil } +// celPlaceholderNames returns the name passed to each placeholder call in e, +// or an empty string where the name is not a string literal. +func celPlaceholderNames(e ast.Expr) []string { + var names []string + ast.PreOrderVisit(e, ast.NewExprVisitor(func(e ast.Expr) { + if !isCELCaddyPlaceholderCall(e) { + return + } + name := "" + if args := e.AsCall().Args(); len(args) == 2 && isCELStringLiteral(args[1]) { + name = args[1].AsLiteral().Value().(string) + } + names = append(names, name) + })) + return names +} + // isCELStringExpr indicates whether the expression is a supported string expression func isCELStringExpr(e ast.Expr) bool { return isCELStringLiteral(e) || isCELCaddyPlaceholderCall(e) || isCELConcatCall(e) diff --git a/modules/caddyhttp/reverseproxy/retries_test.go b/modules/caddyhttp/reverseproxy/retries_test.go index bfd70978c..404b18f6b 100644 --- a/modules/caddyhttp/reverseproxy/retries_test.go +++ b/modules/caddyhttp/reverseproxy/retries_test.go @@ -283,50 +283,57 @@ func minimalHandlerWithRetryMatch(retries int, retryMatch caddyhttp.MatcherSets, // code matching a retry_match expression, the request is retried on the next // upstream func TestResponseRetryStatusCode(t *testing.T) { - // Bad upstream: returns 502 - badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadGateway) - })) - t.Cleanup(badServer.Close) + for _, expr := range []string{ + "{http.reverse_proxy.status_code} in [502, 503]", + `ph(req, "http.reverse_proxy.status_code") in [502, 503]`, + } { + t.Run(expr, func(t *testing.T) { + // Bad upstream: returns 502 + badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + t.Cleanup(badServer.Close) - // Good upstream: returns 200 - goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) - })) - t.Cleanup(goodServer.Close) + // Good upstream: returns 200 + goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) + })) + t.Cleanup(goodServer.Close) - retryMatch := caddyhttp.MatcherSets{ - caddyhttp.MatcherSet{ - newExpressionMatcher(t, "{http.reverse_proxy.status_code} in [502, 503]"), - }, - } + retryMatch := caddyhttp.MatcherSets{ + caddyhttp.MatcherSet{ + newExpressionMatcher(t, expr), + }, + } - // RoundRobin picks index 1 first, then 0 - upstreams := []*Upstream{ - {Host: new(Host), Dial: goodServer.Listener.Addr().String()}, - {Host: new(Host), Dial: badServer.Listener.Addr().String()}, - } + // RoundRobin picks index 1 first, then 0 + upstreams := []*Upstream{ + {Host: new(Host), Dial: goodServer.Listener.Addr().String()}, + {Host: new(Host), Dial: badServer.Listener.Addr().String()}, + } - h := minimalHandlerWithRetryMatch(1, retryMatch, upstreams...) + h := minimalHandlerWithRetryMatch(1, retryMatch, upstreams...) - req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) - req = prepareTestRequest(req) - rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) + req = prepareTestRequest(req) + rec := httptest.NewRecorder() - err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - return nil - })) + err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { + return nil + })) - gotStatus := rec.Code - if err != nil { - if herr, ok := err.(caddyhttp.HandlerError); ok { - gotStatus = herr.StatusCode - } - } + gotStatus := rec.Code + if err != nil { + if herr, ok := err.(caddyhttp.HandlerError); ok { + gotStatus = herr.StatusCode + } + } - if gotStatus != http.StatusOK { - t.Errorf("status: got %d, want %d (err=%v)", gotStatus, http.StatusOK, err) + if gotStatus != http.StatusOK { + t.Errorf("status: got %d, want %d (err=%v)", gotStatus, http.StatusOK, err) + } + }) } } @@ -565,6 +572,64 @@ func TestRequestOnlyMatcherDoesNotRetryResponses(t *testing.T) { } } +// TestRequestOnlyExpressionDoesNotRetryResponses verifies that an expression +// matcher that only tests the request or the selected upstream (no response +// placeholders) does not cause successful responses to be retried +func TestRequestOnlyExpressionDoesNotRetryResponses(t *testing.T) { + for _, expr := range []string{ + `{http.request.method} == "POST"`, + `{http.reverse_proxy.upstream.host} == "127.0.0.1"`, + `{http.request.uri.path} != "\{http.reverse_proxy.status_code}"`, + `{http.reverse_proxy.status_codes} == null`, + } { + t.Run(expr, func(t *testing.T) { + var hits atomic.Int32 + + // Server returns 200 OK for all requests + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) + })) + t.Cleanup(server.Close) + + retryMatch := caddyhttp.MatcherSets{ + caddyhttp.MatcherSet{ + newExpressionMatcher(t, expr), + }, + } + + upstreams := []*Upstream{ + {Host: new(Host), Dial: server.Listener.Addr().String()}, + {Host: new(Host), Dial: server.Listener.Addr().String()}, + } + + h := minimalHandlerWithRetryMatch(2, retryMatch, upstreams...) + + req := httptest.NewRequest(http.MethodPost, "http://example.com/", strings.NewReader("order=1")) + req = prepareTestRequest(req) + rec := httptest.NewRecorder() + + err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { + return nil + })) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if hits.Load() != 1 { + t.Errorf("upstream hits: got %d, want 1 (should not retry successful responses)", hits.Load()) + } + if rec.Code != http.StatusOK { + t.Errorf("status: got %d, want %d", rec.Code, http.StatusOK) + } + if rec.Body.String() != "ok" { + t.Errorf("body: got %q, want %q", rec.Body.String(), "ok") + } + }) + } +} + // brokenUpstreamAddr returns the address of a TCP listener that accepts // connections but immediately closes them, causing a transport error (not // a dial error). This simulates an upstream that is reachable but broken diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index 90f7a4bd8..22a439fb4 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -30,6 +30,7 @@ import ( "net/netip" "net/textproto" "net/url" + "slices" "strconv" "strings" "sync" @@ -1168,14 +1169,13 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe // check if the response matches a retry match entry; if so, // close the body and return a retryable error so the request // is retried with the next upstream. Only evaluate matcher sets - // that contain at least one expression matcher, since those are - // the ones that can reference response data ({rp.status_code}, - // {rp.header.*}). Pure request-only matchers (method, path, etc.) - // are skipped to avoid retrying every response that matches a - // request condition + // with an expression that references response data ({rp.status_code}, + // {rp.header.*}). Request-only matchers (method, path, or an + // expression without response placeholders) are skipped to avoid + // retrying every response that matches a request condition if h.LoadBalancing != nil && len(h.LoadBalancing.RetryMatch) > 0 { for _, matcherSet := range h.LoadBalancing.RetryMatch { - if !matcherSetHasExpressionMatcher(matcherSet) { + if !matcherSetReferencesResponse(matcherSet) { continue } match, err := matcherSet.MatchWithError(req) @@ -2024,15 +2024,37 @@ type RequestHeaderOpsTransport interface { RequestHeaderOps() *headers.HeaderOps } -// matcherSetHasExpressionMatcher reports whether a matcher set contains -// at least one expression matcher. Expression matchers can reference -// response data via placeholders like {rp.status_code}. Matcher sets -// without expression matchers only test request properties and should -// not be evaluated for response-based retry decisions -func matcherSetHasExpressionMatcher(matcherSet caddyhttp.MatcherSet) bool { +// responsePlaceholders are the placeholders that are only set once the +// upstream round trip completes ({rp.status_code} is adapted to +// {http.reverse_proxy.status_code}, and so on) +var responsePlaceholders = []string{ + "http.reverse_proxy.status_code", + "http.reverse_proxy.status_text", + "http.reverse_proxy.upstream.latency", + "http.reverse_proxy.upstream.latency_ms", +} + +// responseHeaderPlaceholderPrefix is the prefix of the response +// header placeholders, {http.reverse_proxy.header.*} +const responseHeaderPlaceholderPrefix = "http.reverse_proxy.header." + +// matcherSetReferencesResponse reports whether a matcher set contains an +// expression matcher that reads response data, or a placeholder whose name +// is only known at evaluation time. Other matcher sets only test request or +// upstream selection properties and should not be evaluated for +// response-based retry decisions +func matcherSetReferencesResponse(matcherSet caddyhttp.MatcherSet) bool { for _, m := range matcherSet { - if _, ok := m.(*caddyhttp.MatchExpression); ok { - return true + expr, ok := m.(*caddyhttp.MatchExpression) + if !ok { + continue + } + for _, name := range expr.Placeholders() { + if name == "" || + slices.Contains(responsePlaceholders, name) || + strings.HasPrefix(name, responseHeaderPlaceholderPrefix) { + return true + } } } return false