From 61f89b734ca75295f1e60a89e5250290dbbd1945 Mon Sep 17 00:00:00 2001 From: sameera Date: Fri, 6 Nov 2026 05:36:35 +0530 Subject: [PATCH 1/2] caddytls: restore SNI conn-policy fast-path, keep first-match order --- modules/caddytls/connpolicy.go | 69 ++++++++++++++++++++++------ modules/caddytls/connpolicy_test.go | 71 +++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 13 deletions(-) diff --git a/modules/caddytls/connpolicy.go b/modules/caddytls/connpolicy.go index 38bc25db0..f3e2c637e 100644 --- a/modules/caddytls/connpolicy.go +++ b/modules/caddytls/connpolicy.go @@ -23,6 +23,7 @@ import ( "encoding/pem" "fmt" "io" + "iter" "os" "reflect" "slices" @@ -99,32 +100,56 @@ func (cp ConnectionPolicies) TLSConfig(ctx caddy.Context) *tls.Config { // using ServerName to match policies is extremely common, especially in configs // with lots and lots of different policies; we can fast-track those by indexing // them by SNI, so we don't have to iterate potentially thousands of policies - // (TODO: this map does not account for wildcards, see if this is a problem in practice? look for reports of high connection latency with wildcard certs but low latency for non-wildcards in multi-thousand-cert deployments) - indexedBySNI := make(map[string]ConnectionPolicies) + // + // only a policy whose SNI matcher lists exact names can be indexed, since it + // can't match any other ServerName; a policy with no SNI matcher, or one whose + // SNI matcher uses a wildcard or a placeholder, may match any name, so it has + // to stay in the candidate set for every lookup (in its original position) to + // preserve first-match order, otherwise an indexed exact-name policy would + // shadow an earlier catch-all drop or a remote_ip/wildcard restriction + indexedBySNI := make(map[string][]int) + var unindexed []int if len(cp) > 30 { - for _, p := range cp { + indexLoop: + for i, p := range cp { for _, m := range p.matchers { - if sni, ok := m.(MatchServerName); ok { - for _, sniName := range sni { - // index for fast lookups during handshakes - indexName := asciiServerNameForMatch(sniName) - indexedBySNI[indexName] = append(indexedBySNI[indexName], p) - } + // provisioned matchers are pointers, but a value may be used + // when policies are built programmatically, so accept both + var sni MatchServerName + switch v := m.(type) { + case *MatchServerName: + sni = *v + case MatchServerName: + sni = v + default: + continue } + if slices.ContainsFunc(sni, func(name string) bool { + return strings.ContainsAny(name, "*{") + }) { + continue + } + for _, sniName := range sni { + // index for fast lookups during handshakes + indexName := asciiServerNameForMatch(sniName) + indexedBySNI[indexName] = append(indexedBySNI[indexName], i) + } + continue indexLoop } + unindexed = append(unindexed, i) } } getConfigForClient := func(hello *tls.ClientHelloInfo) (*tls.Config, error) { // filter policies by SNI first, if possible, to speed things up // when there may be lots of policies - possiblePolicies := cp - if indexedPolicies, ok := indexedBySNI[asciiServerNameForMatch(hello.ServerName)]; ok { - possiblePolicies = indexedPolicies + possiblePolicies := slices.Values(cp) + if indexed, ok := indexedBySNI[asciiServerNameForMatch(hello.ServerName)]; ok { + possiblePolicies = policiesInOrder(cp, indexed, unindexed) } policyLoop: - for _, pol := range possiblePolicies { + for pol := range possiblePolicies { for _, matcher := range pol.matchers { if !matcher.Match(hello) { continue policyLoop @@ -190,6 +215,24 @@ func (cp ConnectionPolicies) TLSConfig(ctx caddy.Context) *tls.Config { return tlsCfg } +// policiesInOrder yields the policies of cp at the indices in a and b, both of +// which must be sorted ascending, in the order they appear in cp. +func policiesInOrder(cp ConnectionPolicies, a, b []int) iter.Seq[*ConnectionPolicy] { + return func(yield func(*ConnectionPolicy) bool) { + for len(a) > 0 || len(b) > 0 { + var i int + if len(b) == 0 || (len(a) > 0 && a[0] < b[0]) { + i, a = a[0], a[1:] + } else { + i, b = b[0], b[1:] + } + if !yield(cp[i]) { + return + } + } + } +} + // ConnectionPolicy specifies the logic for handling a TLS handshake. // An empty policy is valid; safe and sensible defaults will be used. type ConnectionPolicy struct { diff --git a/modules/caddytls/connpolicy_test.go b/modules/caddytls/connpolicy_test.go index 798d476b9..e50f59a3f 100644 --- a/modules/caddytls/connpolicy_test.go +++ b/modules/caddytls/connpolicy_test.go @@ -60,6 +60,77 @@ func TestConnectionPolicyIDNSNIMatcherFastPath(t *testing.T) { } } +func TestConnectionPolicySNIFastPathKeepsFirstMatch(t *testing.T) { + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) + defer cancel() + + anyIP := MatchRemoteIP{Ranges: []string{"0.0.0.0/0", "::/0"}} + if err := anyIP.Provision(ctx); err != nil { + t.Fatalf("provisioning remote_ip matcher: %v", err) + } + + wildcardTLSConfig := &tls.Config{ClientAuth: tls.RequireAndVerifyClientCert} + siteTLSConfig := &tls.Config{} + catchAllTLSConfig := &tls.Config{} + + // each case puts a policy that matches "site.localhost" by something other + // than an exact SNI name ahead of the exact-SNI policy for that host; with + // more than 30 policies the SNI fast-path index must still reach the earlier + // policy first + for i, tc := range []struct { + first *ConnectionPolicy + expect *tls.Config + expectDrop bool + }{ + { + // catch-all drop ahead of the exact-SNI policy + first: &ConnectionPolicy{matchers: []ConnectionMatcher{anyIP}, Drop: true}, + expectDrop: true, + }, + { + // wildcard-SNI policy ahead of the exact-SNI policy + first: &ConnectionPolicy{matchers: []ConnectionMatcher{MatchServerName{"*.localhost"}}, TLSConfig: wildcardTLSConfig}, + expect: wildcardTLSConfig, + }, + { + // placeholder-SNI policy ahead of the exact-SNI policy + first: &ConnectionPolicy{matchers: []ConnectionMatcher{MatchServerName{"{env.CADDY_TEST_UNSET}site.localhost"}}, TLSConfig: wildcardTLSConfig}, + expect: wildcardTLSConfig, + }, + } { + policies := ConnectionPolicies{tc.first} + for j := 0; j < 30; j++ { + policies = append(policies, &ConnectionPolicy{ + matchers: []ConnectionMatcher{MatchServerName{fmt.Sprintf("example-%d.localhost", j)}}, + TLSConfig: &tls.Config{}, + }) + } + policies = append(policies, + &ConnectionPolicy{matchers: []ConnectionMatcher{MatchServerName{"site.localhost"}}, TLSConfig: siteTLSConfig}, + &ConnectionPolicy{TLSConfig: catchAllTLSConfig}, + ) + + hello := &tls.ClientHelloInfo{ + ServerName: "site.localhost", + Conn: testConn{addr: testAddr("198.51.100.1:443")}, + } + got, err := policies.TLSConfig(ctx).GetConfigForClient(hello) + if tc.expectDrop { + if err == nil { + t.Errorf("Test %d: expected the first policy to drop the connection, got a config", i) + } + continue + } + if err != nil { + t.Errorf("Test %d: unexpected error: %v", i, err) + continue + } + if got != tc.expect { + t.Errorf("Test %d: indexed lookup did not return the first matching policy", i) + } + } +} + func TestClientAuthenticationUnmarshalCaddyfileWithDirectiveName(t *testing.T) { const test_der_1 = `MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ==` const test_cert_file_1 = "../../caddytest/caddy.ca.cer" From 86291da6335292b28b6596199821828a018d7223 Mon Sep 17 00:00:00 2001 From: sameera Date: Fri, 13 Nov 2026 13:14:58 +0530 Subject: [PATCH 2/2] caddytls: test SNI fast-path with provisioned matchers, add benchmark Build the fast-path test policies through ConnectionPolicies.Provision so the SNI matchers are the *MatchServerName pointers used at handshake time, and add a benchmark for exact-SNI lookups across 100/1000/10000 policies. --- modules/caddytls/connpolicy_test.go | 125 +++++++++++++++++++--------- 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/modules/caddytls/connpolicy_test.go b/modules/caddytls/connpolicy_test.go index e50f59a3f..8aaeee3ae 100644 --- a/modules/caddytls/connpolicy_test.go +++ b/modules/caddytls/connpolicy_test.go @@ -60,77 +60,122 @@ func TestConnectionPolicyIDNSNIMatcherFastPath(t *testing.T) { } } -func TestConnectionPolicySNIFastPathKeepsFirstMatch(t *testing.T) { - ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) - defer cancel() +// sniPolicy returns an unprovisioned connection policy that matches the +// given server names, configured the same way as one decoded from JSON. +func sniPolicy(names ...string) *ConnectionPolicy { + raw, err := json.Marshal(names) + if err != nil { + panic(err) + } + return &ConnectionPolicy{MatchersRaw: caddy.ModuleMap{"sni": raw}} +} - anyIP := MatchRemoteIP{Ranges: []string{"0.0.0.0/0", "::/0"}} - if err := anyIP.Provision(ctx); err != nil { - t.Fatalf("provisioning remote_ip matcher: %v", err) +func TestConnectionPolicySNIFastPathKeepsFirstMatch(t *testing.T) { + ctx, err := caddy.ProvisionContext(&caddy.Config{}) + if err != nil { + t.Fatal(err) } - wildcardTLSConfig := &tls.Config{ClientAuth: tls.RequireAndVerifyClientCert} - siteTLSConfig := &tls.Config{} - catchAllTLSConfig := &tls.Config{} - - // each case puts a policy that matches "site.localhost" by something other - // than an exact SNI name ahead of the exact-SNI policy for that host; with - // more than 30 policies the SNI fast-path index must still reach the earlier - // policy first + // each case puts a policy ahead of the exact-SNI policy for + // "site.localhost"; with more than 30 provisioned policies the SNI + // fast-path index is used, and it must still select the same policy + // a linear scan would for i, tc := range []struct { - first *ConnectionPolicy - expect *tls.Config - expectDrop bool + first *ConnectionPolicy + wantFirst bool }{ { // catch-all drop ahead of the exact-SNI policy - first: &ConnectionPolicy{matchers: []ConnectionMatcher{anyIP}, Drop: true}, - expectDrop: true, + first: &ConnectionPolicy{ + MatchersRaw: caddy.ModuleMap{"remote_ip": json.RawMessage(`{"ranges":["0.0.0.0/0","::/0"]}`)}, + Drop: true, + }, + wantFirst: true, }, { // wildcard-SNI policy ahead of the exact-SNI policy - first: &ConnectionPolicy{matchers: []ConnectionMatcher{MatchServerName{"*.localhost"}}, TLSConfig: wildcardTLSConfig}, - expect: wildcardTLSConfig, + first: sniPolicy("*.localhost"), + wantFirst: true, }, { // placeholder-SNI policy ahead of the exact-SNI policy - first: &ConnectionPolicy{matchers: []ConnectionMatcher{MatchServerName{"{env.CADDY_TEST_UNSET}site.localhost"}}, TLSConfig: wildcardTLSConfig}, - expect: wildcardTLSConfig, + first: sniPolicy("{env.CADDY_TEST_UNSET}site.localhost"), + wantFirst: true, + }, + { + // unrelated exact-SNI policy ahead of the exact-SNI policy + first: sniPolicy("other.localhost"), + wantFirst: false, }, } { policies := ConnectionPolicies{tc.first} - for j := 0; j < 30; j++ { - policies = append(policies, &ConnectionPolicy{ - matchers: []ConnectionMatcher{MatchServerName{fmt.Sprintf("example-%d.localhost", j)}}, - TLSConfig: &tls.Config{}, - }) + for j := range 30 { + policies = append(policies, sniPolicy(fmt.Sprintf("example-%d.localhost", j))) + } + site := sniPolicy("site.localhost") + policies = append(policies, site, &ConnectionPolicy{}) + if err := policies.Provision(ctx); err != nil { + t.Fatalf("Test %d: provisioning policies: %v", i, err) + } + if _, ok := site.matchers[0].(*MatchServerName); !ok { + t.Fatalf("Test %d: expected provisioned SNI matcher to be *MatchServerName, got %T", i, site.matchers[0]) } - policies = append(policies, - &ConnectionPolicy{matchers: []ConnectionMatcher{MatchServerName{"site.localhost"}}, TLSConfig: siteTLSConfig}, - &ConnectionPolicy{TLSConfig: catchAllTLSConfig}, - ) hello := &tls.ClientHelloInfo{ ServerName: "site.localhost", Conn: testConn{addr: testAddr("198.51.100.1:443")}, } got, err := policies.TLSConfig(ctx).GetConfigForClient(hello) - if tc.expectDrop { + switch { + case tc.first.Drop: if err == nil { t.Errorf("Test %d: expected the first policy to drop the connection, got a config", i) } - continue - } - if err != nil { + case err != nil: t.Errorf("Test %d: unexpected error: %v", i, err) - continue - } - if got != tc.expect { - t.Errorf("Test %d: indexed lookup did not return the first matching policy", i) + case tc.wantFirst && got != tc.first.TLSConfig: + t.Errorf("Test %d: indexed lookup skipped the earlier matching policy", i) + case !tc.wantFirst && got != site.TLSConfig: + t.Errorf("Test %d: indexed lookup did not select the exact-SNI policy", i) } } } +func BenchmarkConnectionPolicySNIFastPath(b *testing.B) { + ctx, err := caddy.ProvisionContext(&caddy.Config{}) + if err != nil { + b.Fatal(err) + } + + for _, n := range []int{100, 1000, 10000} { + b.Run(fmt.Sprintf("policies=%d", n), func(b *testing.B) { + // one exact-SNI policy per site followed by a catch-all, which + // is the shape the Caddyfile adapter produces for many sites + policies := make(ConnectionPolicies, 0, n+1) + for i := range n { + policies = append(policies, sniPolicy(fmt.Sprintf("site-%d.localhost", i))) + } + policies = append(policies, &ConnectionPolicy{}) + if err := policies.Provision(ctx); err != nil { + b.Fatal(err) + } + getConfigForClient := policies.TLSConfig(ctx).GetConfigForClient + + // the last site is the worst case for a linear scan + hello := &tls.ClientHelloInfo{ServerName: fmt.Sprintf("site-%d.localhost", n-1)} + want := policies[n-1].TLSConfig + + b.ReportAllocs() + for b.Loop() { + got, err := getConfigForClient(hello) + if err != nil || got != want { + b.Fatalf("selected wrong policy: %v", err) + } + } + }) + } +} + func TestClientAuthenticationUnmarshalCaddyfileWithDirectiveName(t *testing.T) { const test_der_1 = `MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ==` const test_cert_file_1 = "../../caddytest/caddy.ca.cer"