mirror of
https://github.com/caddyserver/caddy.git
synced 2026-11-21 07:19:40 +00:00
reverseproxy: Only retry responses for expressions that use response data (#8168)
* reverseproxy: Only retry responses for expressions that use response data
Since #7569 every lb_retry_match set with an expression is evaluated against
upstream responses. An expression that only tests the request, such as
{http.request.method} == "POST", then matched every successful response:
the request was sent again to the next upstreams and the client got a 502
once retries ran out. Only treat expressions referencing {rp.*} placeholders
as response conditions.
* reverseproxy: Only count placeholders set from the response as response data
* reverseproxy: Read response placeholders from the checked expression
The expression matcher records the placeholder names its checked CEL
expression reads, so retry_match finds direct ph() calls and ignores
escaped placeholder text in string literals. A placeholder name only
known at evaluation time counts as response data.
* reverseproxy: Match response placeholder names exactly
Only the header placeholders are matched by prefix now, and both
upstream latency placeholders are listed. MatchExpression.Placeholders
returns a copy, so callers cannot change the provisioned matcher.
(cherry picked from commit 7c7456353e)
This commit is contained in:
parent
4c3e75075b
commit
970dc7cf00
3 changed files with 165 additions and 50 deletions
|
|
@ -22,6 +22,7 @@ import (
|
|||
"net/http"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -70,6 +71,7 @@ type MatchExpression struct {
|
|||
Name string `json:"name,omitempty"`
|
||||
|
||||
expandedExpr string
|
||||
placeholders []string
|
||||
prg cel.Program
|
||||
ta types.Adapter
|
||||
|
||||
|
|
@ -192,6 +194,8 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error {
|
|||
return fmt.Errorf("CEL request matcher expects return type of bool, not %s", checked.OutputType())
|
||||
}
|
||||
|
||||
m.placeholders = celPlaceholderNames(checked.NativeRep().Expr())
|
||||
|
||||
// compile the "program"
|
||||
m.prg, err = env.Program(checked, cel.EvalOptions(cel.OptOptimize))
|
||||
if err != nil {
|
||||
|
|
@ -200,6 +204,13 @@ func (m *MatchExpression) Provision(ctx caddy.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Placeholders returns the names of the placeholders the provisioned
|
||||
// expression reads. A name that is only known at evaluation time is
|
||||
// returned as an empty string.
|
||||
func (m MatchExpression) Placeholders() []string {
|
||||
return slices.Clone(m.placeholders)
|
||||
}
|
||||
|
||||
// Match returns true if r matches m.
|
||||
func (m MatchExpression) Match(r *http.Request) bool {
|
||||
match, err := m.MatchWithError(r)
|
||||
|
|
@ -729,6 +740,23 @@ func CELValueToMapStrList(data ref.Val) (map[string][]string, error) {
|
|||
return mapStrListStr, nil
|
||||
}
|
||||
|
||||
// celPlaceholderNames returns the name passed to each placeholder call in e,
|
||||
// or an empty string where the name is not a string literal.
|
||||
func celPlaceholderNames(e ast.Expr) []string {
|
||||
var names []string
|
||||
ast.PreOrderVisit(e, ast.NewExprVisitor(func(e ast.Expr) {
|
||||
if !isCELCaddyPlaceholderCall(e) {
|
||||
return
|
||||
}
|
||||
name := ""
|
||||
if args := e.AsCall().Args(); len(args) == 2 && isCELStringLiteral(args[1]) {
|
||||
name = args[1].AsLiteral().Value().(string)
|
||||
}
|
||||
names = append(names, name)
|
||||
}))
|
||||
return names
|
||||
}
|
||||
|
||||
// isCELStringExpr indicates whether the expression is a supported string expression
|
||||
func isCELStringExpr(e ast.Expr) bool {
|
||||
return isCELStringLiteral(e) || isCELCaddyPlaceholderCall(e) || isCELConcatCall(e)
|
||||
|
|
|
|||
|
|
@ -283,50 +283,57 @@ func minimalHandlerWithRetryMatch(retries int, retryMatch caddyhttp.MatcherSets,
|
|||
// code matching a retry_match expression, the request is retried on the next
|
||||
// upstream
|
||||
func TestResponseRetryStatusCode(t *testing.T) {
|
||||
// Bad upstream: returns 502
|
||||
badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
}))
|
||||
t.Cleanup(badServer.Close)
|
||||
for _, expr := range []string{
|
||||
"{http.reverse_proxy.status_code} in [502, 503]",
|
||||
`ph(req, "http.reverse_proxy.status_code") in [502, 503]`,
|
||||
} {
|
||||
t.Run(expr, func(t *testing.T) {
|
||||
// Bad upstream: returns 502
|
||||
badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
}))
|
||||
t.Cleanup(badServer.Close)
|
||||
|
||||
// Good upstream: returns 200
|
||||
goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
}))
|
||||
t.Cleanup(goodServer.Close)
|
||||
// Good upstream: returns 200
|
||||
goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
}))
|
||||
t.Cleanup(goodServer.Close)
|
||||
|
||||
retryMatch := caddyhttp.MatcherSets{
|
||||
caddyhttp.MatcherSet{
|
||||
newExpressionMatcher(t, "{http.reverse_proxy.status_code} in [502, 503]"),
|
||||
},
|
||||
}
|
||||
retryMatch := caddyhttp.MatcherSets{
|
||||
caddyhttp.MatcherSet{
|
||||
newExpressionMatcher(t, expr),
|
||||
},
|
||||
}
|
||||
|
||||
// RoundRobin picks index 1 first, then 0
|
||||
upstreams := []*Upstream{
|
||||
{Host: new(Host), Dial: goodServer.Listener.Addr().String()},
|
||||
{Host: new(Host), Dial: badServer.Listener.Addr().String()},
|
||||
}
|
||||
// RoundRobin picks index 1 first, then 0
|
||||
upstreams := []*Upstream{
|
||||
{Host: new(Host), Dial: goodServer.Listener.Addr().String()},
|
||||
{Host: new(Host), Dial: badServer.Listener.Addr().String()},
|
||||
}
|
||||
|
||||
h := minimalHandlerWithRetryMatch(1, retryMatch, upstreams...)
|
||||
h := minimalHandlerWithRetryMatch(1, retryMatch, upstreams...)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
req = prepareTestRequest(req)
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
req = prepareTestRequest(req)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
|
||||
return nil
|
||||
}))
|
||||
err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
|
||||
return nil
|
||||
}))
|
||||
|
||||
gotStatus := rec.Code
|
||||
if err != nil {
|
||||
if herr, ok := err.(caddyhttp.HandlerError); ok {
|
||||
gotStatus = herr.StatusCode
|
||||
}
|
||||
}
|
||||
gotStatus := rec.Code
|
||||
if err != nil {
|
||||
if herr, ok := err.(caddyhttp.HandlerError); ok {
|
||||
gotStatus = herr.StatusCode
|
||||
}
|
||||
}
|
||||
|
||||
if gotStatus != http.StatusOK {
|
||||
t.Errorf("status: got %d, want %d (err=%v)", gotStatus, http.StatusOK, err)
|
||||
if gotStatus != http.StatusOK {
|
||||
t.Errorf("status: got %d, want %d (err=%v)", gotStatus, http.StatusOK, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -565,6 +572,64 @@ func TestRequestOnlyMatcherDoesNotRetryResponses(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestRequestOnlyExpressionDoesNotRetryResponses verifies that an expression
|
||||
// matcher that only tests the request or the selected upstream (no response
|
||||
// placeholders) does not cause successful responses to be retried
|
||||
func TestRequestOnlyExpressionDoesNotRetryResponses(t *testing.T) {
|
||||
for _, expr := range []string{
|
||||
`{http.request.method} == "POST"`,
|
||||
`{http.reverse_proxy.upstream.host} == "127.0.0.1"`,
|
||||
`{http.request.uri.path} != "\{http.reverse_proxy.status_code}"`,
|
||||
`{http.reverse_proxy.status_codes} == null`,
|
||||
} {
|
||||
t.Run(expr, func(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
|
||||
// Server returns 200 OK for all requests
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
retryMatch := caddyhttp.MatcherSets{
|
||||
caddyhttp.MatcherSet{
|
||||
newExpressionMatcher(t, expr),
|
||||
},
|
||||
}
|
||||
|
||||
upstreams := []*Upstream{
|
||||
{Host: new(Host), Dial: server.Listener.Addr().String()},
|
||||
{Host: new(Host), Dial: server.Listener.Addr().String()},
|
||||
}
|
||||
|
||||
h := minimalHandlerWithRetryMatch(2, retryMatch, upstreams...)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "http://example.com/", strings.NewReader("order=1"))
|
||||
req = prepareTestRequest(req)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
err := h.ServeHTTP(rec, req, caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
|
||||
return nil
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if hits.Load() != 1 {
|
||||
t.Errorf("upstream hits: got %d, want 1 (should not retry successful responses)", hits.Load())
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("status: got %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
if rec.Body.String() != "ok" {
|
||||
t.Errorf("body: got %q, want %q", rec.Body.String(), "ok")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// brokenUpstreamAddr returns the address of a TCP listener that accepts
|
||||
// connections but immediately closes them, causing a transport error (not
|
||||
// a dial error). This simulates an upstream that is reachable but broken
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
"net/netip"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -1168,14 +1169,13 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
|
|||
// check if the response matches a retry match entry; if so,
|
||||
// close the body and return a retryable error so the request
|
||||
// is retried with the next upstream. Only evaluate matcher sets
|
||||
// that contain at least one expression matcher, since those are
|
||||
// the ones that can reference response data ({rp.status_code},
|
||||
// {rp.header.*}). Pure request-only matchers (method, path, etc.)
|
||||
// are skipped to avoid retrying every response that matches a
|
||||
// request condition
|
||||
// with an expression that references response data ({rp.status_code},
|
||||
// {rp.header.*}). Request-only matchers (method, path, or an
|
||||
// expression without response placeholders) are skipped to avoid
|
||||
// retrying every response that matches a request condition
|
||||
if h.LoadBalancing != nil && len(h.LoadBalancing.RetryMatch) > 0 {
|
||||
for _, matcherSet := range h.LoadBalancing.RetryMatch {
|
||||
if !matcherSetHasExpressionMatcher(matcherSet) {
|
||||
if !matcherSetReferencesResponse(matcherSet) {
|
||||
continue
|
||||
}
|
||||
match, err := matcherSet.MatchWithError(req)
|
||||
|
|
@ -2024,15 +2024,37 @@ type RequestHeaderOpsTransport interface {
|
|||
RequestHeaderOps() *headers.HeaderOps
|
||||
}
|
||||
|
||||
// matcherSetHasExpressionMatcher reports whether a matcher set contains
|
||||
// at least one expression matcher. Expression matchers can reference
|
||||
// response data via placeholders like {rp.status_code}. Matcher sets
|
||||
// without expression matchers only test request properties and should
|
||||
// not be evaluated for response-based retry decisions
|
||||
func matcherSetHasExpressionMatcher(matcherSet caddyhttp.MatcherSet) bool {
|
||||
// responsePlaceholders are the placeholders that are only set once the
|
||||
// upstream round trip completes ({rp.status_code} is adapted to
|
||||
// {http.reverse_proxy.status_code}, and so on)
|
||||
var responsePlaceholders = []string{
|
||||
"http.reverse_proxy.status_code",
|
||||
"http.reverse_proxy.status_text",
|
||||
"http.reverse_proxy.upstream.latency",
|
||||
"http.reverse_proxy.upstream.latency_ms",
|
||||
}
|
||||
|
||||
// responseHeaderPlaceholderPrefix is the prefix of the response
|
||||
// header placeholders, {http.reverse_proxy.header.*}
|
||||
const responseHeaderPlaceholderPrefix = "http.reverse_proxy.header."
|
||||
|
||||
// matcherSetReferencesResponse reports whether a matcher set contains an
|
||||
// expression matcher that reads response data, or a placeholder whose name
|
||||
// is only known at evaluation time. Other matcher sets only test request or
|
||||
// upstream selection properties and should not be evaluated for
|
||||
// response-based retry decisions
|
||||
func matcherSetReferencesResponse(matcherSet caddyhttp.MatcherSet) bool {
|
||||
for _, m := range matcherSet {
|
||||
if _, ok := m.(*caddyhttp.MatchExpression); ok {
|
||||
return true
|
||||
expr, ok := m.(*caddyhttp.MatchExpression)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, name := range expr.Placeholders() {
|
||||
if name == "" ||
|
||||
slices.Contains(responsePlaceholders, name) ||
|
||||
strings.HasPrefix(name, responseHeaderPlaceholderPrefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue