CERBERUS/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2026-05-13 09:36:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

The "multipath" parameter of the "listen" directive.

Browse source 
When configured, it enables Multipath TCP support on a listen socket.
As of now it works on Linux starting with Linux 5.6 and glibc 2.32,
where it is enabled with an IPPROTO_MPTCP socket(2) protocol.

To avoid EADDRINUSE errors in bind() and listen() when transitioning
between sockets with different protocols, SO_REUSEPORT is set on both
sockets.  See f7f1607bf for potential implications.

Based on previous work by Maxime Dourov and Anthony Doeraene.
This commit is contained in:
Sergey Kandaurov 2025-10-16 15:22:56 +00:00 • committed by Sergey Kandaurov
parent a29476464c
commit 920dc099c1
13 changed files with 104 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

2
contrib/vim/syntax/nginx.vim
View file

@ -65,7 +65,7 @@ syn match ngxListenComment '#.*$'
\ contained
\ nextgroup=@ngxListenParams skipwhite skipempty
syn keyword ngxListenOptions contained
\ default_server ssl quic proxy_protocol
\ default_server ssl quic proxy_protocol multipath
\ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind
\ ipv6only reuseport so_keepalive
\ nextgroup=@ngxListenParams skipwhite skipempty

36
src/core/ngx_connection.c
View file

@ -315,6 +315,25 @@ ngx_set_inherited_sockets(ngx_cycle_t *cycle)
continue;
}
#ifdef SO_PROTOCOL
olen = sizeof(int);
if (getsockopt(ls[i].fd, SOL_SOCKET, SO_PROTOCOL,
(void *) &ls[i].protocol, &olen)
== -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
"getsockopt(SO_PROTOCOL) %V failed, ignored",
&ls[i].addr_text);
ls[i].protocol = 0;
} else if (ls[i].protocol == IPPROTO_TCP) {
ls[i].protocol = 0;
}
#endif
#if (NGX_HAVE_TCP_FASTOPEN)
olen = sizeof(int);
@ -436,12 +455,16 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
#if (NGX_HAVE_REUSEPORT)
if (ls[i].add_reuseport) {
if (ls[i].add_reuseport || ls[i].change_protocol) {
/*
* to allow transition from a socket without SO_REUSEPORT
* to multiple sockets with SO_REUSEPORT, we have to set
* SO_REUSEPORT on the old socket before opening new ones
*
* to allow transition between different socket protocols
* (e.g. IPPROTO_MPTCP), SO_REUSEPORT is set on both old
* and new sockets
*/
int reuseport = 1;
@ -474,7 +497,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
}
#endif
if (ls[i].fd != (ngx_socket_t) -1) {
if (ls[i].fd != (ngx_socket_t) -1 && !ls[i].change_protocol) {
continue;
}
@ -487,7 +510,8 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
continue;
}
s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0);
s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type,
ls[i].protocol);
if (s == (ngx_socket_t) -1) {
ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
@ -517,7 +541,9 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
#if (NGX_HAVE_REUSEPORT)
if (ls[i].reuseport && !ngx_test_config) {
if ((ls[i].reuseport || ls[i].change_protocol)
&& !ngx_test_config)
{
int reuseport;
reuseport = 1;
@ -651,6 +677,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
if (ls[i].type != SOCK_STREAM) {
ls[i].fd = s;
ls[i].open = 1;
continue;
}
@ -689,6 +716,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
ls[i].listen = 1;
ls[i].fd = s;
ls[i].open = 1;
}
if (!failed) {

3
src/core/ngx_connection.h
View file

@ -24,6 +24,7 @@ struct ngx_listening_s {
ngx_str_t addr_text;
int type;
int protocol;
int backlog;
int rcvbuf;
@ -75,6 +76,8 @@ struct ngx_listening_s {
unsigned keepalive:2;
unsigned quic:1;
unsigned change_protocol:1;
unsigned deferred_accept:1;
unsigned delete_deferred:1;
unsigned add_deferred:1;

18
src/core/ngx_cycle.c
View file

@ -535,9 +535,15 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
== NGX_OK)
{
nls[n].fd = ls[i].fd;
nls[n].inherited = ls[i].inherited;
nls[n].previous = &ls[i];
ls[i].remain = 1;
if (ls[i].protocol != nls[n].protocol) {
nls[n].change_protocol = 1;
} else {
nls[n].inherited = ls[i].inherited;
ls[i].remain = 1;
}
if (ls[i].backlog != nls[n].backlog) {
nls[n].listen = 1;
@ -590,7 +596,6 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
}
if (nls[n].fd == (ngx_socket_t) -1) {
nls[n].open = 1;
#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
if (nls[n].accept_filter) {
nls[n].add_deferred = 1;
@ -605,20 +610,21 @@ ngx_init_cycle(ngx_cycle_t *old_cycle)
}
} else {
#if (NGX_HAVE_DEFERRED_ACCEPT)
ls = cycle->listening.elts;
for (i = 0; i < cycle->listening.nelts; i++) {
ls[i].open = 1;
#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
#ifdef SO_ACCEPTFILTER
if (ls[i].accept_filter) {
ls[i].add_deferred = 1;
}
#endif
#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT)
#ifdef TCP_DEFER_ACCEPT
if (ls[i].deferred_accept) {
ls[i].add_deferred = 1;
}
#endif
}
#endif
}
if (ngx_open_listening_sockets(cycle) != NGX_OK) {

1
src/http/ngx_http.c
View file

@ -1846,6 +1846,7 @@ ngx_http_add_listening(ngx_conf_t *cf, ngx_http_conf_addr_t *addr)
#endif
ls->type = addr->opt.type;
ls->protocol = addr->opt.protocol;
ls->backlog = addr->opt.backlog;
ls->rcvbuf = addr->opt.rcvbuf;
ls->sndbuf = addr->opt.sndbuf;

19
src/http/ngx_http_core_module.c
View file

@ -4223,6 +4223,19 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
continue;
}
if (ngx_strcmp(value[n].data, "multipath") == 0) {
#ifdef IPPROTO_MPTCP
lsopt.protocol = IPPROTO_MPTCP;
lsopt.set = 1;
lsopt.bind = 1;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"multipath is not supported "
"on this platform, ignored");
#endif
continue;
}
if (ngx_strcmp(value[n].data, "ssl") == 0) {
#if (NGX_HTTP_SSL)
lsopt.ssl = 1;
@ -4389,6 +4402,12 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
#endif
#ifdef IPPROTO_MPTCP
if (lsopt.protocol == IPPROTO_MPTCP) {
return "\"multipath\" parameter is incompatible with \"quic\"";
}
#endif
#if (NGX_HTTP_SSL)
if (lsopt.ssl) {
return "\"ssl\" parameter is incompatible with \"quic\"";

1
src/http/ngx_http_core_module.h
View file

@ -88,6 +88,7 @@ typedef struct {
int rcvbuf;
int sndbuf;
int type;
int protocol;
#if (NGX_HAVE_SETFIB)
int setfib;
#endif

1
src/mail/ngx_mail.c
View file

@ -332,6 +332,7 @@ ngx_mail_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports)
ls->log.data = &ls->addr_text;
ls->log.handler = ngx_accept_log_error;
ls->protocol = addr[i].opt.protocol;
ls->backlog = addr[i].opt.backlog;
ls->rcvbuf = addr[i].opt.rcvbuf;
ls->sndbuf = addr[i].opt.sndbuf;

1
src/mail/ngx_mail.h
View file

@ -47,6 +47,7 @@ typedef struct {
int tcp_keepintvl;
int tcp_keepcnt;
#endif
int protocol;
int backlog;
int rcvbuf;
int sndbuf;

12
src/mail/ngx_mail_core_module.c
View file

@ -447,6 +447,18 @@ ngx_mail_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
#endif
}
if (ngx_strcmp(value[i].data, "multipath") == 0) {
#ifdef IPPROTO_MPTCP
ls->protocol = IPPROTO_MPTCP;
ls->bind = 1;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"multipath is not supported "
"on this platform, ignored");
#endif
continue;
}
if (ngx_strcmp(value[i].data, "ssl") == 0) {
#if (NGX_MAIL_SSL)
ngx_mail_ssl_conf_t *sslcf;

1
src/stream/ngx_stream.c
View file

@ -1010,6 +1010,7 @@ ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr)
ls->log.handler = ngx_accept_log_error;
ls->type = addr->opt.type;
ls->protocol = addr->opt.protocol;
ls->backlog = addr->opt.backlog;
ls->rcvbuf = addr->opt.rcvbuf;
ls->sndbuf = addr->opt.sndbuf;

1
src/stream/ngx_stream.h
View file

@ -62,6 +62,7 @@ typedef struct {
int rcvbuf;
int sndbuf;
int type;
int protocol;
#if (NGX_HAVE_SETFIB)
int setfib;
#endif

19
src/stream/ngx_stream_core_module.c
View file

@ -1202,6 +1202,19 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
continue;
}
if (ngx_strcmp(value[i].data, "multipath") == 0) {
#ifdef IPPROTO_MPTCP
lsopt.protocol = IPPROTO_MPTCP;
lsopt.set = 1;
lsopt.bind = 1;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"multipath is not supported "
"on this platform, ignored");
#endif
continue;
}
if (ngx_strcmp(value[i].data, "ssl") == 0) {
#if (NGX_STREAM_SSL)
lsopt.ssl = 1;
@ -1338,6 +1351,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
#endif
#ifdef IPPROTO_MPTCP
if (lsopt.protocol == IPPROTO_MPTCP) {
return "\"multipath\" parameter is incompatible with \"udp\"";
}
#endif
#if (NGX_STREAM_SSL)
if (lsopt.ssl) {
return "\"ssl\" parameter is incompatible with \"udp\"";