reverseproxy: Only retry responses for expressions that use response data (#8168)

* reverseproxy: Only retry responses for expressions that use response data

Since #7569 every lb_retry_match set with an expression is evaluated against
upstream responses. An expression that only tests the request, such as
{http.request.method} == "POST", then matched every successful response:
the request was sent again to the next upstreams and the client got a 502
once retries ran out. Only treat expressions referencing {rp.*} placeholders
as response conditions.

* reverseproxy: Only count placeholders set from the response as response data

* reverseproxy: Read response placeholders from the checked expression

The expression matcher records the placeholder names its checked CEL
expression reads, so retry_match finds direct ph() calls and ignores
escaped placeholder text in string literals. A placeholder name only
known at evaluation time counts as response data.

* reverseproxy: Match response placeholder names exactly

Only the header placeholders are matched by prefix now, and both
upstream latency placeholders are listed. MatchExpression.Placeholders
returns a copy, so callers cannot change the provisioned matcher.

(cherry picked from commit 7c7456353e)
This commit is contained in:
Jay Hemnani 2026-10-10 02:52:32 +05:30 • committed by Francis Lavoie
parent 4c3e75075b
commit 970dc7cf00
No known key found for this signature in database
3 changed files with 165 additions and 50 deletions

View file

@ -22,6 +22,7 @@ import (
"net/http"
"reflect"
"regexp"
"slices"
"strings"
"time"
@ -70,6 +71,7 @@ type MatchExpression struct {
Name string `json:"name,omitempty"`
expandedExpr string
placeholders []string
prg cel.Program
ta types.Adapter
@ -192,6 +194,8 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error {
return fmt.Errorf("CEL request matcher expects return type of bool, not %s", checked.OutputType())
}
m.placeholders = celPlaceholderNames(checked.NativeRep().Expr())
// compile the "program"
m.prg, err = env.Program(checked, cel.EvalOptions(cel.OptOptimize))
if err != nil {
@ -200,6 +204,13 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error {
return nil
}
// Placeholders returns the names of the placeholders the provisioned
// expression reads. A name that is only known at evaluation time is
// returned as an empty string.
func (m MatchExpression) Placeholders() []string {
return slices.Clone(m.placeholders)
}
// Match returns true if r matches m.
func (m MatchExpression) Match(r *http.Request) bool {
match, err := m.MatchWithError(r)
@ -729,6 +740,23 @@ func CELValueToMapStrList(data ref.Val) (map[string][]string, error) {
return mapStrListStr, nil
}
// celPlaceholderNames returns the name passed to each placeholder call in e,
// or an empty string where the name is not a string literal.
func celPlaceholderNames(e ast.Expr) []string {
var names []string
ast.PreOrderVisit(e, ast.NewExprVisitor(func(e ast.Expr) {
if !isCELCaddyPlaceholderCall(e) {
return
}
name := ""
if args := e.AsCall().Args(); len(args) == 2 && isCELStringLiteral(args[1]) {
name = args[1].AsLiteral().Value().(string)
}
names = append(names, name)
}))
return names
}
// isCELStringExpr indicates whether the expression is a supported string expression
func isCELStringExpr(e ast.Expr) bool {
return isCELStringLiteral(e) || isCELCaddyPlaceholderCall(e) || isCELConcatCall(e)

View file

@ -283,50 +283,57 @@ func minimalHandlerWithRetryMatch(retries int, retryMatch caddyhttp.MatcherSets,
// code matching a retry_match expression, the request is retried on the next
// upstream
func TestResponseRetryStatusCode(t *testing.T) {
// Bad upstream: returns 502
badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadGateway)
}))
t.Cleanup(badServer.Close)
for _, expr := range []string{
"{http.reverse_proxy.status_code} in [502, 503]",
`ph(req, "http.reverse_proxy.status_code") in [502, 503]`,
} {
t.Run(expr, func(t *testing.T) {
// Bad upstream: returns 502
badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadGateway)
}))
t.Cleanup(badServer.Close)
// Good upstream: returns 200
goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}))
t.Cleanup(goodServer.Close)
// Good upstream: returns 200
goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}))
t.Cleanup(goodServer.Close)
retryMatch := caddyhttp.MatcherSets{
caddyhttp.MatcherSet{
newExpressionMatcher(t, "{http.reverse_proxy.status_code} in [502, 503]"),
},
}
retryMatch := caddyhttp.MatcherSets{
caddyhttp.MatcherSet{
newExpressionMatcher(t, expr),
},
}
// RoundRobin picks index 1 first, then 0
upstreams := []*Upstream{
{Host: new(Host), Dial: goodServer.Listener.Addr().String()},
{Host: new(Host), Dial: badServer.Listener.Addr().String()},
}
// RoundRobin picks index 1 first, then 0
upstreams := []*Upstream{
{Host: new(Host), Dial: goodServer.Listener.Addr().String()},
{Host: new(Host), Dial: badServer.Listener.Addr().String()},
}
h := minimalHandlerWithRetryMatch(1, retryMatch, upstreams...)
h := minimalHandlerWithRetryMatch(1, retryMatch, upstreams...)
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
req = prepareTestRequest(req)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
req = prepareTestRequest(req)
rec := httptest.NewRecorder()
err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
return nil
}))
err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
return nil
}))
gotStatus := rec.Code
if err != nil {
if herr, ok := err.(caddyhttp.HandlerError); ok {
gotStatus = herr.StatusCode
}
}
gotStatus := rec.Code
if err != nil {
if herr, ok := err.(caddyhttp.HandlerError); ok {
gotStatus = herr.StatusCode
}
}
if gotStatus != http.StatusOK {
t.Errorf("status: got %d, want %d (err=%v)", gotStatus, http.StatusOK, err)
if gotStatus != http.StatusOK {
t.Errorf("status: got %d, want %d (err=%v)", gotStatus, http.StatusOK, err)
}
})
}
}
@ -565,6 +572,64 @@ func TestRequestOnlyMatcherDoesNotRetryResponses(t *testing.T) {
}
}
// TestRequestOnlyExpressionDoesNotRetryResponses verifies that an expression
// matcher that only tests the request or the selected upstream (no response
// placeholders) does not cause successful responses to be retried
func TestRequestOnlyExpressionDoesNotRetryResponses(t *testing.T) {
for _, expr := range []string{
`{http.request.method} == "POST"`,
`{http.reverse_proxy.upstream.host} == "127.0.0.1"`,
`{http.request.uri.path} != "\{http.reverse_proxy.status_code}"`,
`{http.reverse_proxy.status_codes} == null`,
} {
t.Run(expr, func(t *testing.T) {
var hits atomic.Int32
// Server returns 200 OK for all requests
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}))
t.Cleanup(server.Close)
retryMatch := caddyhttp.MatcherSets{
caddyhttp.MatcherSet{
newExpressionMatcher(t, expr),
},
}
upstreams := []*Upstream{
{Host: new(Host), Dial: server.Listener.Addr().String()},
{Host: new(Host), Dial: server.Listener.Addr().String()},
}
h := minimalHandlerWithRetryMatch(2, retryMatch, upstreams...)
req := httptest.NewRequest(http.MethodPost, "http://example.com/", strings.NewReader("order=1"))
req = prepareTestRequest(req)
rec := httptest.NewRecorder()
err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
return nil
}))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if hits.Load() != 1 {
t.Errorf("upstream hits: got %d, want 1 (should not retry successful responses)", hits.Load())
}
if rec.Code != http.StatusOK {
t.Errorf("status: got %d, want %d", rec.Code, http.StatusOK)
}
if rec.Body.String() != "ok" {
t.Errorf("body: got %q, want %q", rec.Body.String(), "ok")
}
})
}
}
// brokenUpstreamAddr returns the address of a TCP listener that accepts
// connections but immediately closes them, causing a transport error (not
// a dial error). This simulates an upstream that is reachable but broken

View file

@ -30,6 +30,7 @@ import (
"net/netip"
"net/textproto"
"net/url"
"slices"
"strconv"
"strings"
"sync"
@ -1168,14 +1169,13 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
// check if the response matches a retry match entry; if so,
// close the body and return a retryable error so the request
// is retried with the next upstream. Only evaluate matcher sets
// that contain at least one expression matcher, since those are
// the ones that can reference response data ({rp.status_code},
// {rp.header.*}). Pure request-only matchers (method, path, etc.)
// are skipped to avoid retrying every response that matches a
// request condition
// with an expression that references response data ({rp.status_code},
// {rp.header.*}). Request-only matchers (method, path, or an
// expression without response placeholders) are skipped to avoid
// retrying every response that matches a request condition
if h.LoadBalancing != nil && len(h.LoadBalancing.RetryMatch) > 0 {
for _, matcherSet := range h.LoadBalancing.RetryMatch {
if !matcherSetHasExpressionMatcher(matcherSet) {
if !matcherSetReferencesResponse(matcherSet) {
continue
}
match, err := matcherSet.MatchWithError(req)
@ -2024,15 +2024,37 @@ type RequestHeaderOpsTransport interface {
RequestHeaderOps() *headers.HeaderOps
}
// matcherSetHasExpressionMatcher reports whether a matcher set contains
// at least one expression matcher. Expression matchers can reference
// response data via placeholders like {rp.status_code}. Matcher sets
// without expression matchers only test request properties and should
// not be evaluated for response-based retry decisions
func matcherSetHasExpressionMatcher(matcherSet caddyhttp.MatcherSet) bool {
// responsePlaceholders are the placeholders that are only set once the
// upstream round trip completes ({rp.status_code} is adapted to
// {http.reverse_proxy.status_code}, and so on)
var responsePlaceholders = []string{
"http.reverse_proxy.status_code",
"http.reverse_proxy.status_text",
"http.reverse_proxy.upstream.latency",
"http.reverse_proxy.upstream.latency_ms",
}
// responseHeaderPlaceholderPrefix is the prefix of the response
// header placeholders, {http.reverse_proxy.header.*}
const responseHeaderPlaceholderPrefix = "http.reverse_proxy.header."
// matcherSetReferencesResponse reports whether a matcher set contains an
// expression matcher that reads response data, or a placeholder whose name
// is only known at evaluation time. Other matcher sets only test request or
// upstream selection properties and should not be evaluated for
// response-based retry decisions
func matcherSetReferencesResponse(matcherSet caddyhttp.MatcherSet) bool {
for _, m := range matcherSet {
if _, ok := m.(*caddyhttp.MatchExpression); ok {
return true
expr, ok := m.(*caddyhttp.MatchExpression)
if !ok {
continue
}
for _, name := range expr.Placeholders() {
if name == "" ||
slices.Contains(responsePlaceholders, name) ||
strings.HasPrefix(name, responseHeaderPlaceholderPrefix) {
return true
}
}
}
return false