From 16618f454ca6f78f37dfd85a90573d8ffd43c1c8 Mon Sep 17 00:00:00 2001 From: sam-kota Date: Wed, 18 Nov 2026 13:55:48 +0530 Subject: [PATCH] caddytls: restore SNI conn-policy fast-path, keep first-match order (#8373) * caddytls: restore SNI conn-policy fast-path, keep first-match order * caddytls: test SNI fast-path with provisioned matchers, add benchmark Build the fast-path test policies through ConnectionPolicies.Provision so the SNI matchers are the *MatchServerName pointers used at handshake time, and add a benchmark for exact-SNI lookups across 100/1000/10000 policies. --- modules/caddytls/connpolicy.go | 69 +++++++++++++---- modules/caddytls/connpolicy_test.go | 116 ++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 13 deletions(-) diff --git a/modules/caddytls/connpolicy.go b/modules/caddytls/connpolicy.go index 38bc25db0..f3e2c637e 100644 --- a/modules/caddytls/connpolicy.go +++ b/modules/caddytls/connpolicy.go @@ -23,6 +23,7 @@ import ( "encoding/pem" "fmt" "io" + "iter" "os" "reflect" "slices" @@ -99,32 +100,56 @@ func (cp ConnectionPolicies) TLSConfig(ctx caddy.Context) *tls.Config { // using ServerName to match policies is extremely common, especially in configs // with lots and lots of different policies; we can fast-track those by indexing // them by SNI, so we don't have to iterate potentially thousands of policies - // (TODO: this map does not account for wildcards, see if this is a problem in practice? look for reports of high connection latency with wildcard certs but low latency for non-wildcards in multi-thousand-cert deployments) - indexedBySNI := make(map[string]ConnectionPolicies) + // + // only a policy whose SNI matcher lists exact names can be indexed, since it + // can't match any other ServerName; a policy with no SNI matcher, or one whose + // SNI matcher uses a wildcard or a placeholder, may match any name, so it has + // to stay in the candidate set for every lookup (in its original position) to + // preserve first-match order, otherwise an indexed exact-name policy would + // shadow an earlier catch-all drop or a remote_ip/wildcard restriction + indexedBySNI := make(map[string][]int) + var unindexed []int if len(cp) > 30 { - for _, p := range cp { + indexLoop: + for i, p := range cp { for _, m := range p.matchers { - if sni, ok := m.(MatchServerName); ok { - for _, sniName := range sni { - // index for fast lookups during handshakes - indexName := asciiServerNameForMatch(sniName) - indexedBySNI[indexName] = append(indexedBySNI[indexName], p) - } + // provisioned matchers are pointers, but a value may be used + // when policies are built programmatically, so accept both + var sni MatchServerName + switch v := m.(type) { + case *MatchServerName: + sni = *v + case MatchServerName: + sni = v + default: + continue } + if slices.ContainsFunc(sni, func(name string) bool { + return strings.ContainsAny(name, "*{") + }) { + continue + } + for _, sniName := range sni { + // index for fast lookups during handshakes + indexName := asciiServerNameForMatch(sniName) + indexedBySNI[indexName] = append(indexedBySNI[indexName], i) + } + continue indexLoop } + unindexed = append(unindexed, i) } } getConfigForClient := func(hello *tls.ClientHelloInfo) (*tls.Config, error) { // filter policies by SNI first, if possible, to speed things up // when there may be lots of policies - possiblePolicies := cp - if indexedPolicies, ok := indexedBySNI[asciiServerNameForMatch(hello.ServerName)]; ok { - possiblePolicies = indexedPolicies + possiblePolicies := slices.Values(cp) + if indexed, ok := indexedBySNI[asciiServerNameForMatch(hello.ServerName)]; ok { + possiblePolicies = policiesInOrder(cp, indexed, unindexed) } policyLoop: - for _, pol := range possiblePolicies { + for pol := range possiblePolicies { for _, matcher := range pol.matchers { if !matcher.Match(hello) { continue policyLoop @@ -190,6 +215,24 @@ func (cp ConnectionPolicies) TLSConfig(ctx caddy.Context) *tls.Config { return tlsCfg } +// policiesInOrder yields the policies of cp at the indices in a and b, both of +// which must be sorted ascending, in the order they appear in cp. +func policiesInOrder(cp ConnectionPolicies, a, b []int) iter.Seq[*ConnectionPolicy] { + return func(yield func(*ConnectionPolicy) bool) { + for len(a) > 0 || len(b) > 0 { + var i int + if len(b) == 0 || (len(a) > 0 && a[0] < b[0]) { + i, a = a[0], a[1:] + } else { + i, b = b[0], b[1:] + } + if !yield(cp[i]) { + return + } + } + } +} + // ConnectionPolicy specifies the logic for handling a TLS handshake. // An empty policy is valid; safe and sensible defaults will be used. type ConnectionPolicy struct { diff --git a/modules/caddytls/connpolicy_test.go b/modules/caddytls/connpolicy_test.go index 798d476b9..8aaeee3ae 100644 --- a/modules/caddytls/connpolicy_test.go +++ b/modules/caddytls/connpolicy_test.go @@ -60,6 +60,122 @@ func TestConnectionPolicyIDNSNIMatcherFastPath(t *testing.T) { } } +// sniPolicy returns an unprovisioned connection policy that matches the +// given server names, configured the same way as one decoded from JSON. +func sniPolicy(names ...string) *ConnectionPolicy { + raw, err := json.Marshal(names) + if err != nil { + panic(err) + } + return &ConnectionPolicy{MatchersRaw: caddy.ModuleMap{"sni": raw}} +} + +func TestConnectionPolicySNIFastPathKeepsFirstMatch(t *testing.T) { + ctx, err := caddy.ProvisionContext(&caddy.Config{}) + if err != nil { + t.Fatal(err) + } + + // each case puts a policy ahead of the exact-SNI policy for + // "site.localhost"; with more than 30 provisioned policies the SNI + // fast-path index is used, and it must still select the same policy + // a linear scan would + for i, tc := range []struct { + first *ConnectionPolicy + wantFirst bool + }{ + { + // catch-all drop ahead of the exact-SNI policy + first: &ConnectionPolicy{ + MatchersRaw: caddy.ModuleMap{"remote_ip": json.RawMessage(`{"ranges":["0.0.0.0/0","::/0"]}`)}, + Drop: true, + }, + wantFirst: true, + }, + { + // wildcard-SNI policy ahead of the exact-SNI policy + first: sniPolicy("*.localhost"), + wantFirst: true, + }, + { + // placeholder-SNI policy ahead of the exact-SNI policy + first: sniPolicy("{env.CADDY_TEST_UNSET}site.localhost"), + wantFirst: true, + }, + { + // unrelated exact-SNI policy ahead of the exact-SNI policy + first: sniPolicy("other.localhost"), + wantFirst: false, + }, + } { + policies := ConnectionPolicies{tc.first} + for j := range 30 { + policies = append(policies, sniPolicy(fmt.Sprintf("example-%d.localhost", j))) + } + site := sniPolicy("site.localhost") + policies = append(policies, site, &ConnectionPolicy{}) + if err := policies.Provision(ctx); err != nil { + t.Fatalf("Test %d: provisioning policies: %v", i, err) + } + if _, ok := site.matchers[0].(*MatchServerName); !ok { + t.Fatalf("Test %d: expected provisioned SNI matcher to be *MatchServerName, got %T", i, site.matchers[0]) + } + + hello := &tls.ClientHelloInfo{ + ServerName: "site.localhost", + Conn: testConn{addr: testAddr("198.51.100.1:443")}, + } + got, err := policies.TLSConfig(ctx).GetConfigForClient(hello) + switch { + case tc.first.Drop: + if err == nil { + t.Errorf("Test %d: expected the first policy to drop the connection, got a config", i) + } + case err != nil: + t.Errorf("Test %d: unexpected error: %v", i, err) + case tc.wantFirst && got != tc.first.TLSConfig: + t.Errorf("Test %d: indexed lookup skipped the earlier matching policy", i) + case !tc.wantFirst && got != site.TLSConfig: + t.Errorf("Test %d: indexed lookup did not select the exact-SNI policy", i) + } + } +} + +func BenchmarkConnectionPolicySNIFastPath(b *testing.B) { + ctx, err := caddy.ProvisionContext(&caddy.Config{}) + if err != nil { + b.Fatal(err) + } + + for _, n := range []int{100, 1000, 10000} { + b.Run(fmt.Sprintf("policies=%d", n), func(b *testing.B) { + // one exact-SNI policy per site followed by a catch-all, which + // is the shape the Caddyfile adapter produces for many sites + policies := make(ConnectionPolicies, 0, n+1) + for i := range n { + policies = append(policies, sniPolicy(fmt.Sprintf("site-%d.localhost", i))) + } + policies = append(policies, &ConnectionPolicy{}) + if err := policies.Provision(ctx); err != nil { + b.Fatal(err) + } + getConfigForClient := policies.TLSConfig(ctx).GetConfigForClient + + // the last site is the worst case for a linear scan + hello := &tls.ClientHelloInfo{ServerName: fmt.Sprintf("site-%d.localhost", n-1)} + want := policies[n-1].TLSConfig + + b.ReportAllocs() + for b.Loop() { + got, err := getConfigForClient(hello) + if err != nil || got != want { + b.Fatalf("selected wrong policy: %v", err) + } + } + }) + } +} + func TestClientAuthenticationUnmarshalCaddyfileWithDirectiveName(t *testing.T) { const test_der_1 = `MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ==` const test_cert_file_1 = "../../caddytest/caddy.ca.cer"