diff --git a/auto/options b/auto/options index 271153a76..6f14aa5e3 100644 --- a/auto/options +++ b/auto/options @@ -44,7 +44,7 @@ EVENT_POLL=NO USE_THREADS=NO NGX_FILE_AIO=NO - +NGX_IO_URING=NO QUIC_BPF=NO HTTP=YES @@ -218,6 +218,7 @@ do --with-threads) USE_THREADS=YES ;; --with-file-aio) NGX_FILE_AIO=YES ;; + --with-io_uring) NGX_IO_URING=YES ;; --without-quic_bpf_module) QUIC_BPF=NONE ;; @@ -455,6 +456,7 @@ cat << END --with-threads enable thread pool support --with-file-aio enable file AIO support + --with-io_uring enable io_uring support --without-quic_bpf_module disable ngx_quic_bpf_module diff --git a/auto/sources b/auto/sources index 46408ee53..864fb1bbc 100644 --- a/auto/sources +++ b/auto/sources @@ -127,6 +127,8 @@ IOCP_SRCS=src/event/modules/ngx_iocp_module.c FILE_AIO_SRCS="src/os/unix/ngx_file_aio_read.c" LINUX_AIO_SRCS="src/os/unix/ngx_linux_aio_read.c" +LINUX_IO_URING_SRCS="src/os/unix/ngx_linux_io_uring_read.c" + UNIX_INCS="$CORE_INCS $EVENT_INCS src/os/unix" UNIX_DEPS="$CORE_DEPS $EVENT_DEPS \ diff --git a/auto/unix b/auto/unix index 6087e04fd..617112c8d 100644 --- a/auto/unix +++ b/auto/unix @@ -543,6 +543,15 @@ ngx_feature_libs= ngx_feature_test="accept4(0, NULL, NULL, SOCK_NONBLOCK)" . auto/feature +if [ "$NGX_FILE_AIO" = YES -a "$NGX_IO_URING" = YES ]; then + cat << END + +$0: --with-io_uring and --with-file-aio cannot be enabled together yet + +END + exit 1 +fi + if [ $NGX_FILE_AIO = YES ]; then ngx_feature="kqueue AIO support" @@ -614,6 +623,80 @@ END fi fi +if [ "$NGX_IO_URING" = YES ]; then + + if [ "$NGX_SYSTEM" != "Linux" ]; then + cat << END + +$0: io_uring is supported on Linux only + +END + exit 1 + fi + + ngx_feature="io_uring headers and syscalls" + ngx_feature_name="NGX_HAVE_IO_URING" + ngx_feature_run=no + ngx_feature_incs="#include + #include + #include + #include +#ifndef SYS_io_uring_setup +#define SYS_io_uring_setup __NR_io_uring_setup +#endif +#ifndef SYS_io_uring_enter +#define SYS_io_uring_enter __NR_io_uring_enter +#endif +#ifndef SYS_io_uring_register +#define SYS_io_uring_register __NR_io_uring_register +#endif" + ngx_feature_path= + ngx_feature_libs= + ngx_feature_test="struct io_uring_params p; + memset(&p, 0, sizeof(p)); + (void) IORING_OP_READ; + (void) IORING_OP_READV; + (void) IORING_REGISTER_EVENTFD; + (void) syscall(SYS_io_uring_setup, 2, &p)" + . auto/feature + + if [ $ngx_found = no ]; then + cat << END + +$0: io_uring was requested but not found +need linux/io_uring.h and SYS_io_uring_* declarations + +END + exit 1 + fi + + ngx_feature="liburing library" + ngx_feature_name="NGX_HAVE_LIBURING" + ngx_feature_run=no + ngx_feature_incs="#include " + ngx_feature_path= + ngx_feature_libs="-luring" + ngx_feature_test="struct io_uring ring; + (void) io_uring_queue_init(2, &ring, 0); + io_uring_queue_exit(&ring)" + . auto/feature + + if [ $ngx_found = no ]; then + cat << END + +$0: io_uring was requested but liburing was not found +install liburing (headers and library), or disable --with-io_uring + +END + exit 1 + fi + + CORE_LIBS="$CORE_LIBS -luring" + + CORE_SRCS="$CORE_SRCS $LINUX_IO_URING_SRCS" +fi + + have=NGX_HAVE_UNIX_DOMAIN . auto/have diff --git a/src/core/ngx_core.h b/src/core/ngx_core.h index 02890b843..6c922b79d 100644 --- a/src/core/ngx_core.h +++ b/src/core/ngx_core.h @@ -23,6 +23,7 @@ typedef struct ngx_command_s ngx_command_t; typedef struct ngx_file_s ngx_file_t; typedef struct ngx_event_s ngx_event_t; typedef struct ngx_event_aio_s ngx_event_aio_t; +typedef struct ngx_event_io_uring_s ngx_event_io_uring_t; typedef struct ngx_connection_s ngx_connection_t; typedef struct ngx_thread_task_s ngx_thread_task_t; typedef struct ngx_ssl_s ngx_ssl_t; diff --git a/src/core/ngx_file.h b/src/core/ngx_file.h index 320adc295..64047564f 100644 --- a/src/core/ngx_file.h +++ b/src/core/ngx_file.h @@ -34,6 +34,10 @@ struct ngx_file_s { ngx_event_aio_t *aio; #endif +#if (NGX_HAVE_IO_URING || NGX_COMPAT) + ngx_event_io_uring_t *io_uring; +#endif + unsigned valid_info:1; unsigned directio:1; }; diff --git a/src/core/ngx_module.h b/src/core/ngx_module.h index a415cd6d9..cc952405e 100644 --- a/src/core/ngx_module.h +++ b/src/core/ngx_module.h @@ -35,7 +35,7 @@ #define NGX_MODULE_SIGNATURE_2 "0" #endif -#if (NGX_HAVE_FILE_AIO || NGX_COMPAT) +#if (NGX_HAVE_FILE_AIO || NGX_HAVE_IO_URING || NGX_COMPAT) #define NGX_MODULE_SIGNATURE_3 "1" #else #define NGX_MODULE_SIGNATURE_3 "0" diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c index 98e3ce7c8..5358c3cd6 100644 --- a/src/event/modules/ngx_epoll_module.c +++ b/src/event/modules/ngx_epoll_module.c @@ -123,7 +123,7 @@ static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler); static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING || NGX_HAVE_FILE_AIO) static void ngx_epoll_eventfd_handler(ngx_event_t *ev); #endif @@ -140,7 +140,15 @@ static ngx_event_t notify_event; static ngx_connection_t notify_conn; #endif -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING) + +int ngx_eventfd = -1; +struct io_uring *ngx_io_uring_ring = NULL; + +static ngx_event_t ngx_eventfd_event; +static ngx_connection_t ngx_eventfd_conn; + +#elif (NGX_HAVE_FILE_AIO) int ngx_eventfd = -1; aio_context_t ngx_aio_ctx = 0; @@ -215,7 +223,101 @@ ngx_module_t ngx_epoll_module = { }; -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING) +static void +ngx_epoll_io_uring_init(ngx_cycle_t *cycle, ngx_epoll_conf_t *epcf) +{ + int n; + int ret; + struct epoll_event ee; +#if (NGX_HAVE_SYS_EVENTFD_H) + ngx_eventfd = eventfd(0, 0); +#else + ngx_eventfd = syscall(SYS_eventfd, 0); +#endif + + if (ngx_eventfd == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "eventfd() failed"); + ngx_io_uring = 0; + return; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "eventfd: %d", ngx_eventfd); + + n = 1; + + if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "ioctl(eventfd, FIONBIO) failed"); + goto failed; + } + + if (ngx_io_uring_ring == NULL) { + ngx_io_uring_ring = ngx_palloc(cycle->pool, sizeof(struct io_uring)); + if (ngx_io_uring_ring == NULL) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "ngx_palloc failed"); + goto failed; + } + } + + ret = io_uring_queue_init(epcf->aio_requests, ngx_io_uring_ring, 0); + if (ret < 0) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, -ret, + "io_uring_queue_init() failed"); + goto free_ring; + } + + ret = io_uring_register_eventfd(ngx_io_uring_ring, ngx_eventfd); + if (ret < 0) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, -ret, + "io_uring_register_eventfd() failed"); + goto exit_ring; + } + + ngx_eventfd_event.data = &ngx_eventfd_conn; + ngx_eventfd_event.handler = ngx_epoll_eventfd_handler; + ngx_eventfd_event.log = cycle->log; + ngx_eventfd_event.active = 1; + ngx_eventfd_conn.fd = ngx_eventfd; + ngx_eventfd_conn.read = &ngx_eventfd_event; + ngx_eventfd_conn.log = cycle->log; + + ee.events = EPOLLIN|EPOLLET; + ee.data.ptr = &ngx_eventfd_conn; + + if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) { + return; + } + + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed"); + goto exit_ring; + +exit_ring: + + if (ngx_io_uring_ring) { + io_uring_queue_exit(ngx_io_uring_ring); + } + +free_ring: + + ngx_pfree(cycle->pool, ngx_io_uring_ring); + ngx_io_uring_ring = NULL; + +failed: + + if (close(ngx_eventfd) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "eventfd close() failed"); + } + + ngx_eventfd = -1; + ngx_io_uring = 0; +} +#elif (NGX_HAVE_FILE_AIO) /* * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly @@ -341,7 +443,9 @@ ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer) } #endif -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING) + ngx_epoll_io_uring_init(cycle, epcf); +#elif (NGX_HAVE_FILE_AIO) ngx_epoll_aio_init(cycle, epcf); #endif @@ -547,7 +651,22 @@ ngx_epoll_done(ngx_cycle_t *cycle) #endif -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING) + if (ngx_io_uring_ring) { + io_uring_queue_exit(ngx_io_uring_ring); + ngx_io_uring_ring = NULL; + } + + if (ngx_eventfd != -1) { + if (close(ngx_eventfd) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "eventfd close() failed"); + } + + ngx_eventfd = -1; + } + +#elif (NGX_HAVE_FILE_AIO) if (ngx_eventfd != -1) { @@ -935,8 +1054,80 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) return NGX_OK; } +#if (NGX_HAVE_IO_URING) -#if (NGX_HAVE_FILE_AIO) +static void +ngx_epoll_eventfd_handler(ngx_event_t *ev) +{ + int ret; + ssize_t n; + uint64_t ready; + ngx_err_t err; + ngx_event_t *e; + ngx_event_io_uring_t *io_uring; + struct io_uring_cqe *cqe; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_uring eventfd handler"); + + n = read(ngx_eventfd, &ready, 8); + + err = ngx_errno; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %z", n); + + if (n != 8) { + if (n == -1) { + if (err == NGX_EAGAIN) { + return; + } + + ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed"); + return; + } + + ngx_log_error(NGX_LOG_ALERT, ev->log, 0, + "read(eventfd) returned only %z bytes", n); + return; + } + + for ( ;; ) { + ret = io_uring_peek_cqe(ngx_io_uring_ring, &cqe); + + if (ret == -EAGAIN) { + return; + } + + if (ret < 0) { + ngx_log_error(NGX_LOG_ALERT, ev->log, -ret, + "io_uring_peek_cqe() failed"); + return; + } + + if (cqe == NULL) { + return; + } + + e = io_uring_cqe_get_data(cqe); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_uring cqe: %p %d", e, cqe->res); + + if (e != NULL) { + e->complete = 1; + e->active = 0; + e->ready = 1; + + io_uring = e->data; + io_uring->res = cqe->res; + + ngx_post_event(e, &ngx_posted_events); + } + + io_uring_cqe_seen(ngx_io_uring_ring, cqe); + } +} +#elif (NGX_HAVE_FILE_AIO) static void ngx_epoll_eventfd_handler(ngx_event_t *ev) diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index deac04e7b..3238e3685 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -163,6 +163,22 @@ struct ngx_event_aio_s { #endif +#if (NGX_HAVE_IO_URING) + +struct ngx_event_io_uring_s { + void *data; + ngx_event_handler_pt handler; + ngx_file_t *file; + + ngx_fd_t fd; + int64_t res; + + ngx_event_t event; +}; + +#endif + + typedef struct { ngx_int_t (*add)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); ngx_int_t (*del)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index 53ddf39bb..de692005c 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -60,6 +60,8 @@ static char *ngx_http_core_limit_except(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_http_core_set_io_uring(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); static char *ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_core_error_page(ngx_conf_t *cf, ngx_command_t *cmd, @@ -423,6 +425,13 @@ static ngx_command_t ngx_http_core_commands[] = { 0, NULL }, + { ngx_string("io_uring"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_core_set_io_uring, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + { ngx_string("aio_write"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, @@ -3651,6 +3660,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf) clcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE; clcf->subrequest_output_buffer_size = NGX_CONF_UNSET_SIZE; clcf->aio = NGX_CONF_UNSET; + clcf->io_uring = NGX_CONF_UNSET; clcf->aio_write = NGX_CONF_UNSET; #if (NGX_THREADS) clcf->thread_pool = NGX_CONF_UNSET_PTR; @@ -3880,6 +3890,7 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->subrequest_output_buffer_size, (size_t) ngx_pagesize); ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF); + ngx_conf_merge_value(conf->io_uring, prev->io_uring, 0); ngx_conf_merge_value(conf->aio_write, prev->aio_write, 0); #if (NGX_THREADS) ngx_conf_merge_ptr_value(conf->thread_pool, prev->thread_pool, NULL); @@ -4888,6 +4899,40 @@ ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +static char * +ngx_http_core_set_io_uring(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_core_loc_conf_t *clcf = conf; + + ngx_str_t *value; + + if (clcf->io_uring != NGX_CONF_UNSET) { + return "is duplicate"; + } + + value = cf->args->elts; + + if (ngx_strcmp(value[1].data, "off") == 0) { + clcf->io_uring = 0; + return NGX_CONF_OK; + } + + if (ngx_strcmp(value[1].data, "on") == 0) { +#if (NGX_HAVE_IO_URING) + clcf->io_uring = 1; + return NGX_CONF_OK; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"io_uring on\" " + "is unsupported on this platform"); + return NGX_CONF_ERROR; +#endif + } + + return "invalid value"; +} + + static char * ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index a13d7ade5..1c09e8ccc 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -397,6 +397,7 @@ struct ngx_http_core_loc_conf_s { ngx_flag_t internal; /* internal */ ngx_flag_t sendfile; /* sendfile */ ngx_flag_t aio; /* aio */ + ngx_flag_t io_uring; /* io_uring */ ngx_flag_t aio_write; /* aio_write */ ngx_flag_t tcp_nopush; /* tcp_nopush */ ngx_flag_t tcp_nodelay; /* tcp_nodelay */ diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index 5209f003b..50d8abd0a 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -18,8 +18,11 @@ static ngx_int_t ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c); static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c); -static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r, +static ssize_t ngx_http_file_cache_async_read(ngx_http_request_t *r, ngx_http_cache_t *c); +#if (NGX_HAVE_IO_URING) +static void ngx_http_cache_io_uring_event_handler(ngx_event_t *ev); +#endif #if (NGX_HAVE_FILE_AIO) static void ngx_http_cache_aio_event_handler(ngx_event_t *ev); #endif @@ -551,7 +554,7 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) ngx_http_file_cache_t *cache; ngx_http_file_cache_header_t *h; - n = ngx_http_file_cache_aio_read(r, c); + n = ngx_http_file_cache_async_read(r, c); if (n < 0) { return n; @@ -681,15 +684,41 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) static ssize_t -ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c) +ngx_http_file_cache_async_read(ngx_http_request_t *r, ngx_http_cache_t *c) { -#if (NGX_HAVE_FILE_AIO || NGX_THREADS) +#if (NGX_HAVE_FILE_AIO || NGX_HAVE_IO_URING || NGX_THREADS) ssize_t n; ngx_http_core_loc_conf_t *clcf; clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); #endif +#if (NGX_HAVE_IO_URING) + + if (clcf->io_uring && ngx_io_uring) { + n = ngx_file_io_uring_read(&c->file, c->buf->pos, c->body_start, 0, + r->pool); + + if (n != NGX_AGAIN) { + c->reading = 0; + return n; + } + + c->reading = 1; + + c->file.io_uring->data = r; + c->file.io_uring->handler = ngx_http_cache_io_uring_event_handler; + + ngx_add_timer(&c->file.io_uring->event, 60000); + + r->main->blocked++; + r->aio = 1; + + return NGX_AGAIN; + } + +#endif + #if (NGX_HAVE_FILE_AIO) if (clcf->aio == NGX_HTTP_AIO_ON && ngx_file_aio) { @@ -736,6 +765,56 @@ ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c) } +#if (NGX_HAVE_IO_URING) + +static void +ngx_http_cache_io_uring_event_handler(ngx_event_t *ev) +{ + ngx_event_io_uring_t *io_uring; + ngx_connection_t *c; + ngx_http_request_t *r; + + io_uring = ev->data; + r = io_uring->data; + c = r->connection; + + ngx_http_set_log_request(c->log, r); + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http file cache io_uring: \"%V?%V\"", + &r->uri, &r->args); + + if (ev->timedout) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "io_uring operation took too long"); + ev->timedout = 0; + return; + } + + if (ev->timer_set) { + ngx_del_timer(ev); + } + + r->main->blocked--; + r->aio = 0; + + if (r->main->terminated) { + /* + * trigger connection event handler if the request was + * terminated + */ + + c->write->handler(c->write); + + } else { + r->write_event_handler(r); + ngx_http_run_posted_requests(c); + } +} + +#endif + + #if (NGX_HAVE_FILE_AIO) static void diff --git a/src/os/unix/ngx_files.c b/src/os/unix/ngx_files.c index 2fec1ece1..82fa1c2b5 100644 --- a/src/os/unix/ngx_files.c +++ b/src/os/unix/ngx_files.c @@ -20,7 +20,11 @@ static ssize_t ngx_writev_file(ngx_file_t *file, ngx_iovec_t *vec, off_t offset); -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING) + +ngx_uint_t ngx_io_uring = 1; + +#elif (NGX_HAVE_FILE_AIO) ngx_uint_t ngx_file_aio = 1; diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h index d084713b6..0468717ec 100644 --- a/src/os/unix/ngx_files.h +++ b/src/os/unix/ngx_files.h @@ -375,7 +375,15 @@ off_t ngx_fs_available(u_char *name); #define ngx_set_stderr_n "dup2(STDERR_FILENO)" -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_IO_URING) + +ngx_int_t ngx_file_io_uring_init(ngx_file_t *file, ngx_pool_t *pool); +ssize_t ngx_file_io_uring_read(ngx_file_t *file, u_char *buf, size_t size, + off_t offset, ngx_pool_t *pool); + +extern ngx_uint_t ngx_io_uring; + +#elif (NGX_HAVE_FILE_AIO) ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool); ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h index d99358c93..6db3acd6a 100644 --- a/src/os/unix/ngx_linux_config.h +++ b/src/os/unix/ngx_linux_config.h @@ -97,6 +97,11 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size); #include #endif #include + +#if (NGX_HAVE_LIBURING) +#include +#endif + #if (NGX_HAVE_FILE_AIO) #include typedef struct iocb ngx_aiocb_t; diff --git a/src/os/unix/ngx_linux_io_uring_read.c b/src/os/unix/ngx_linux_io_uring_read.c new file mode 100644 index 000000000..67c3f4dca --- /dev/null +++ b/src/os/unix/ngx_linux_io_uring_read.c @@ -0,0 +1,165 @@ +/* + * Copyright (C) Nginx, Inc. + */ + + +#include +#include +#include + + +extern struct io_uring *ngx_io_uring_ring; + + +static void ngx_file_io_uring_event_handler(ngx_event_t *ev); + + +ngx_int_t +ngx_file_io_uring_init(ngx_file_t *file, ngx_pool_t *pool) +{ + ngx_event_io_uring_t *io_uring; + + io_uring = ngx_pcalloc(pool, sizeof(ngx_event_io_uring_t)); + if (io_uring == NULL) { + return NGX_ERROR; + } + + io_uring->file = file; + io_uring->fd = file->fd; + io_uring->event.data = io_uring; + io_uring->event.ready = 1; + io_uring->event.log = file->log; + + file->io_uring = io_uring; + + return NGX_OK; +} + + +ssize_t +ngx_file_io_uring_read(ngx_file_t *file, u_char *buf, size_t size, + off_t offset, ngx_pool_t *pool) +{ + int ret; + ngx_err_t err; + ngx_event_t *ev; + ngx_event_io_uring_t *io_uring; + struct io_uring_sqe *sqe; + + if (!ngx_io_uring || ngx_io_uring_ring == NULL) { + return ngx_read_file(file, buf, size, offset); + } + + if (file->io_uring == NULL + && ngx_file_io_uring_init(file, pool) != NGX_OK) + { + return NGX_ERROR; + } + + io_uring = file->io_uring; + ev = &io_uring->event; + + if (!ev->ready) { + ngx_log_error(NGX_LOG_ALERT, file->log, 0, + "second io_uring post for \"%V\"", &file->name); + return NGX_AGAIN; + } + + ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, + "io_uring complete:%d @%O:%uz %V", + ev->complete, offset, size, &file->name); + + if (ev->complete) { + ev->active = 0; + ev->complete = 0; + + if (io_uring->res >= 0) { + ngx_set_errno(0); + return io_uring->res; + } + + err = (ngx_err_t) -io_uring->res; + ngx_set_errno(err); + + ngx_log_error(NGX_LOG_CRIT, file->log, err, + "io_uring read \"%s\" failed", file->name.data); + + if (err == NGX_EINVAL || err == NGX_ENOSYS + || err == NGX_EOPNOTSUPP) + { + ngx_io_uring = 0; + return ngx_read_file(file, buf, size, offset); + } + + return NGX_ERROR; + } + + sqe = io_uring_get_sqe(ngx_io_uring_ring); + + if (sqe == NULL) { + (void) io_uring_submit(ngx_io_uring_ring); + sqe = io_uring_get_sqe(ngx_io_uring_ring); + } + + if (sqe == NULL) { + ngx_log_error(NGX_LOG_ALERT, file->log, 0, + "io_uring_get_sqe() failed"); + return ngx_read_file(file, buf, size, offset); + } + + io_uring_prep_read(sqe, file->fd, buf, size, offset); + io_uring_sqe_set_data(sqe, ev); + + ev->handler = ngx_file_io_uring_event_handler; + + ret = io_uring_submit(ngx_io_uring_ring); + + if (ret > 0) { + ev->active = 1; + ev->ready = 0; + ev->complete = 0; + + return NGX_AGAIN; + } + + if (ret < 0) { + err = (ngx_err_t) -ret; + + if (err == NGX_EAGAIN) { + return ngx_read_file(file, buf, size, offset); + } + + ngx_log_error(NGX_LOG_CRIT, file->log, err, + "io_uring_submit(\"%V\") failed", &file->name); + + if (err == NGX_ENOSYS || err == NGX_EINVAL + || err == NGX_EOPNOTSUPP) + { + ngx_io_uring = 0; + return ngx_read_file(file, buf, size, offset); + } + + return NGX_ERROR; + } + + ngx_log_error(NGX_LOG_ALERT, file->log, 0, + "io_uring_submit(\"%V\") submitted no requests", + &file->name); + + return ngx_read_file(file, buf, size, offset); +} + + +static void +ngx_file_io_uring_event_handler(ngx_event_t *ev) +{ + ngx_event_io_uring_t *io_uring; + + io_uring = ev->data; + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0, + "io_uring event handler fd:%d %V", + io_uring->fd, &io_uring->file->name); + + io_uring->handler(ev); +}