CERBERUS/nginx
1
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2026-05-13 09:36:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge 229cb51c56 into 319b4ea63f

Browse source
This commit is contained in:
Cheng Lingfei 2026-05-12 14:46:18 -04:00 • committed by GitHub
commit 4b75817bae
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 620 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

4
auto/options
View file

@ -44,7 +44,7 @@ EVENT_POLL=NO
USE_THREADS=NO
NGX_FILE_AIO=NO
NGX_IO_URING=NO
QUIC_BPF=NO
HTTP=YES
@ -221,6 +221,7 @@ do
--with-threads) USE_THREADS=YES ;;
--with-file-aio) NGX_FILE_AIO=YES ;;
--with-io_uring) NGX_IO_URING=YES ;;
--without-quic_bpf_module) QUIC_BPF=NONE ;;
@ -469,6 +470,7 @@ cat << END
--with-threads enable thread pool support
--with-file-aio enable file AIO support
--with-io_uring enable io_uring support
--without-quic_bpf_module disable ngx_quic_bpf_module

2
auto/sources
View file

@ -127,6 +127,8 @@ IOCP_SRCS=src/event/modules/ngx_iocp_module.c
FILE_AIO_SRCS="src/os/unix/ngx_file_aio_read.c"
LINUX_AIO_SRCS="src/os/unix/ngx_linux_aio_read.c"
LINUX_IO_URING_SRCS="src/os/unix/ngx_linux_io_uring_read.c"
UNIX_INCS="$CORE_INCS $EVENT_INCS src/os/unix"
UNIX_DEPS="$CORE_DEPS $EVENT_DEPS \

83
auto/unix
View file

@ -543,6 +543,15 @@ ngx_feature_libs=
ngx_feature_test="accept4(0, NULL, NULL, SOCK_NONBLOCK)"
. auto/feature
if [ "$NGX_FILE_AIO" = YES -a "$NGX_IO_URING" = YES ]; then
cat << END
$0: --with-io_uring and --with-file-aio cannot be enabled together yet
END
exit 1
fi
if [ $NGX_FILE_AIO = YES ]; then
ngx_feature="kqueue AIO support"
@ -614,6 +623,80 @@ END
fi
fi
if [ "$NGX_IO_URING" = YES ]; then
if [ "$NGX_SYSTEM" != "Linux" ]; then
cat << END
$0: io_uring is supported on Linux only
END
exit 1
fi
ngx_feature="io_uring headers and syscalls"
ngx_feature_name="NGX_HAVE_IO_URING"
ngx_feature_run=no
ngx_feature_incs="#include <unistd.h>
#include <string.h>
#include <sys/syscall.h>
#include <linux/io_uring.h>
#ifndef SYS_io_uring_setup
#define SYS_io_uring_setup __NR_io_uring_setup
#endif
#ifndef SYS_io_uring_enter
#define SYS_io_uring_enter __NR_io_uring_enter
#endif
#ifndef SYS_io_uring_register
#define SYS_io_uring_register __NR_io_uring_register
#endif"
ngx_feature_path=
ngx_feature_libs=
ngx_feature_test="struct io_uring_params p;
memset(&p, 0, sizeof(p));
(void) IORING_OP_READ;
(void) IORING_OP_READV;
(void) IORING_REGISTER_EVENTFD;
(void) syscall(SYS_io_uring_setup, 2, &p)"
. auto/feature
if [ $ngx_found = no ]; then
cat << END
$0: io_uring was requested but not found
need linux/io_uring.h and SYS_io_uring_* declarations
END
exit 1
fi
ngx_feature="liburing library"
ngx_feature_name="NGX_HAVE_LIBURING"
ngx_feature_run=no
ngx_feature_incs="#include <liburing.h>"
ngx_feature_path=
ngx_feature_libs="-luring"
ngx_feature_test="struct io_uring ring;
(void) io_uring_queue_init(2, &ring, 0);
io_uring_queue_exit(&ring)"
. auto/feature
if [ $ngx_found = no ]; then
cat << END
$0: io_uring was requested but liburing was not found
install liburing (headers and library), or disable --with-io_uring
END
exit 1
fi
CORE_LIBS="$CORE_LIBS -luring"
CORE_SRCS="$CORE_SRCS $LINUX_IO_URING_SRCS"
fi
have=NGX_HAVE_UNIX_DOMAIN . auto/have

1
src/core/ngx_core.h
View file

@ -23,6 +23,7 @@ typedef struct ngx_command_s ngx_command_t;
typedef struct ngx_file_s ngx_file_t;
typedef struct ngx_event_s ngx_event_t;
typedef struct ngx_event_aio_s ngx_event_aio_t;
typedef struct ngx_event_io_uring_s ngx_event_io_uring_t;
typedef struct ngx_connection_s ngx_connection_t;
typedef struct ngx_thread_task_s ngx_thread_task_t;
typedef struct ngx_ssl_s ngx_ssl_t;

4
src/core/ngx_file.h
View file

@ -34,6 +34,10 @@ struct ngx_file_s {
ngx_event_aio_t *aio;
#endif
#if (NGX_HAVE_IO_URING || NGX_COMPAT)
ngx_event_io_uring_t *io_uring;
#endif
unsigned valid_info:1;
unsigned directio:1;
};

2
src/core/ngx_module.h
View file

@ -35,7 +35,7 @@
#define NGX_MODULE_SIGNATURE_2 "0"
#endif
#if (NGX_HAVE_FILE_AIO || NGX_COMPAT)
#if (NGX_HAVE_FILE_AIO || NGX_HAVE_IO_URING || NGX_COMPAT)
#define NGX_MODULE_SIGNATURE_3 "1"
#else
#define NGX_MODULE_SIGNATURE_3 "0"

203
src/event/modules/ngx_epoll_module.c
View file

@ -123,7 +123,7 @@ static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler);
static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
ngx_uint_t flags);
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING || NGX_HAVE_FILE_AIO)
static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
#endif
@ -140,7 +140,15 @@ static ngx_event_t notify_event;
static ngx_connection_t notify_conn;
#endif
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING)
int ngx_eventfd = -1;
struct io_uring *ngx_io_uring_ring = NULL;
static ngx_event_t ngx_eventfd_event;
static ngx_connection_t ngx_eventfd_conn;
#elif (NGX_HAVE_FILE_AIO)
int ngx_eventfd = -1;
aio_context_t ngx_aio_ctx = 0;
@ -215,7 +223,101 @@ ngx_module_t ngx_epoll_module = {
};
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING)
static void
ngx_epoll_io_uring_init(ngx_cycle_t *cycle, ngx_epoll_conf_t *epcf)
{
int n;
int ret;
struct epoll_event ee;
#if (NGX_HAVE_SYS_EVENTFD_H)
ngx_eventfd = eventfd(0, 0);
#else
ngx_eventfd = syscall(SYS_eventfd, 0);
#endif
if (ngx_eventfd == -1) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
"eventfd() failed");
ngx_io_uring = 0;
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"eventfd: %d", ngx_eventfd);
n = 1;
if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
"ioctl(eventfd, FIONBIO) failed");
goto failed;
}
if (ngx_io_uring_ring == NULL) {
ngx_io_uring_ring = ngx_palloc(cycle->pool, sizeof(struct io_uring));
if (ngx_io_uring_ring == NULL) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
"ngx_palloc failed");
goto failed;
}
}
ret = io_uring_queue_init(epcf->aio_requests, ngx_io_uring_ring, 0);
if (ret < 0) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, -ret,
"io_uring_queue_init() failed");
goto free_ring;
}
ret = io_uring_register_eventfd(ngx_io_uring_ring, ngx_eventfd);
if (ret < 0) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, -ret,
"io_uring_register_eventfd() failed");
goto exit_ring;
}
ngx_eventfd_event.data = &ngx_eventfd_conn;
ngx_eventfd_event.handler = ngx_epoll_eventfd_handler;
ngx_eventfd_event.log = cycle->log;
ngx_eventfd_event.active = 1;
ngx_eventfd_conn.fd = ngx_eventfd;
ngx_eventfd_conn.read = &ngx_eventfd_event;
ngx_eventfd_conn.log = cycle->log;
ee.events = EPOLLIN|EPOLLET;
ee.data.ptr = &ngx_eventfd_conn;
if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) {
return;
}
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
"epoll_ctl(EPOLL_CTL_ADD, eventfd) failed");
goto exit_ring;
exit_ring:
if (ngx_io_uring_ring) {
io_uring_queue_exit(ngx_io_uring_ring);
}
free_ring:
ngx_pfree(cycle->pool, ngx_io_uring_ring);
ngx_io_uring_ring = NULL;
failed:
if (close(ngx_eventfd) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"eventfd close() failed");
}
ngx_eventfd = -1;
ngx_io_uring = 0;
}
#elif (NGX_HAVE_FILE_AIO)
/*
* We call io_setup(), io_destroy() io_submit(), and io_getevents() directly
@ -341,7 +443,9 @@ ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
}
#endif
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING)
ngx_epoll_io_uring_init(cycle, epcf);
#elif (NGX_HAVE_FILE_AIO)
ngx_epoll_aio_init(cycle, epcf);
#endif
@ -547,7 +651,22 @@ ngx_epoll_done(ngx_cycle_t *cycle)
#endif
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING)
if (ngx_io_uring_ring) {
io_uring_queue_exit(ngx_io_uring_ring);
ngx_io_uring_ring = NULL;
}
if (ngx_eventfd != -1) {
if (close(ngx_eventfd) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"eventfd close() failed");
}
ngx_eventfd = -1;
}
#elif (NGX_HAVE_FILE_AIO)
if (ngx_eventfd != -1) {
@ -935,8 +1054,80 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
return NGX_OK;
}
#if (NGX_HAVE_IO_URING)
#if (NGX_HAVE_FILE_AIO)
static void
ngx_epoll_eventfd_handler(ngx_event_t *ev)
{
int ret;
ssize_t n;
uint64_t ready;
ngx_err_t err;
ngx_event_t *e;
ngx_event_io_uring_t *io_uring;
struct io_uring_cqe *cqe;
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"io_uring eventfd handler");
n = read(ngx_eventfd, &ready, 8);
err = ngx_errno;
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %z", n);
if (n != 8) {
if (n == -1) {
if (err == NGX_EAGAIN) {
return;
}
ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed");
return;
}
ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
"read(eventfd) returned only %z bytes", n);
return;
}
for ( ;; ) {
ret = io_uring_peek_cqe(ngx_io_uring_ring, &cqe);
if (ret == -EAGAIN) {
return;
}
if (ret < 0) {
ngx_log_error(NGX_LOG_ALERT, ev->log, -ret,
"io_uring_peek_cqe() failed");
return;
}
if (cqe == NULL) {
return;
}
e = io_uring_cqe_get_data(cqe);
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"io_uring cqe: %p %d", e, cqe->res);
if (e != NULL) {
e->complete = 1;
e->active = 0;
e->ready = 1;
io_uring = e->data;
io_uring->res = cqe->res;
ngx_post_event(e, &ngx_posted_events);
}
io_uring_cqe_seen(ngx_io_uring_ring, cqe);
}
}
#elif (NGX_HAVE_FILE_AIO)
static void
ngx_epoll_eventfd_handler(ngx_event_t *ev)

16
src/event/ngx_event.h
View file

@ -163,6 +163,22 @@ struct ngx_event_aio_s {
#endif
#if (NGX_HAVE_IO_URING)
struct ngx_event_io_uring_s {
void *data;
ngx_event_handler_pt handler;
ngx_file_t *file;
ngx_fd_t fd;
int64_t res;
ngx_event_t event;
};
#endif
typedef struct {
ngx_int_t (*add)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
ngx_int_t (*del)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);

45
src/http/ngx_http_core_module.c
View file

@ -60,6 +60,8 @@ static char *ngx_http_core_limit_except(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static char *ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static char *ngx_http_core_set_io_uring(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static char *ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static char *ngx_http_core_error_page(ngx_conf_t *cf, ngx_command_t *cmd,
@ -423,6 +425,13 @@ static ngx_command_t ngx_http_core_commands[] = {
0,
NULL },
{ ngx_string("io_uring"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_http_core_set_io_uring,
NGX_HTTP_LOC_CONF_OFFSET,
0,
NULL },
{ ngx_string("aio_write"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
ngx_conf_set_flag_slot,
@ -3674,6 +3683,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf)
clcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE;
clcf->subrequest_output_buffer_size = NGX_CONF_UNSET_SIZE;
clcf->aio = NGX_CONF_UNSET;
clcf->io_uring = NGX_CONF_UNSET;
clcf->aio_write = NGX_CONF_UNSET;
#if (NGX_THREADS)
clcf->thread_pool = NGX_CONF_UNSET_PTR;
@ -3903,6 +3913,7 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
prev->subrequest_output_buffer_size,
(size_t) ngx_pagesize);
ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF);
ngx_conf_merge_value(conf->io_uring, prev->io_uring, 0);
ngx_conf_merge_value(conf->aio_write, prev->aio_write, 0);
#if (NGX_THREADS)
ngx_conf_merge_ptr_value(conf->thread_pool, prev->thread_pool, NULL);
@ -4911,6 +4922,40 @@ ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
}
static char *
ngx_http_core_set_io_uring(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
ngx_http_core_loc_conf_t *clcf = conf;
ngx_str_t *value;
if (clcf->io_uring != NGX_CONF_UNSET) {
return "is duplicate";
}
value = cf->args->elts;
if (ngx_strcmp(value[1].data, "off") == 0) {
clcf->io_uring = 0;
return NGX_CONF_OK;
}
if (ngx_strcmp(value[1].data, "on") == 0) {
#if (NGX_HAVE_IO_URING)
clcf->io_uring = 1;
return NGX_CONF_OK;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"\"io_uring on\" "
"is unsupported on this platform");
return NGX_CONF_ERROR;
#endif
}
return "invalid value";
}
static char *
ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{

1
src/http/ngx_http_core_module.h
View file

@ -397,6 +397,7 @@ struct ngx_http_core_loc_conf_s {
ngx_flag_t internal; /* internal */
ngx_flag_t sendfile; /* sendfile */
ngx_flag_t aio; /* aio */
ngx_flag_t io_uring; /* io_uring */
ngx_flag_t aio_write; /* aio_write */
ngx_flag_t tcp_nopush; /* tcp_nopush */
ngx_flag_t tcp_nodelay; /* tcp_nodelay */

87
src/http/ngx_http_file_cache.c
View file

@ -18,8 +18,11 @@ static ngx_int_t ngx_http_file_cache_lock_wait(ngx_http_request_t *r,
ngx_http_cache_t *c);
static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r,
ngx_http_cache_t *c);
static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r,
static ssize_t ngx_http_file_cache_async_read(ngx_http_request_t *r,
ngx_http_cache_t *c);
#if (NGX_HAVE_IO_URING)
static void ngx_http_cache_io_uring_event_handler(ngx_event_t *ev);
#endif
#if (NGX_HAVE_FILE_AIO)
static void ngx_http_cache_aio_event_handler(ngx_event_t *ev);
#endif
@ -551,7 +554,7 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c)
ngx_http_file_cache_t *cache;
ngx_http_file_cache_header_t *h;
n = ngx_http_file_cache_aio_read(r, c);
n = ngx_http_file_cache_async_read(r, c);
if (n < 0) {
return n;
@ -681,15 +684,41 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c)
static ssize_t
ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c)
ngx_http_file_cache_async_read(ngx_http_request_t *r, ngx_http_cache_t *c)
{
#if (NGX_HAVE_FILE_AIO || NGX_THREADS)
#if (NGX_HAVE_FILE_AIO || NGX_HAVE_IO_URING || NGX_THREADS)
ssize_t n;
ngx_http_core_loc_conf_t *clcf;
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
#endif
#if (NGX_HAVE_IO_URING)
if (clcf->io_uring && ngx_io_uring) {
n = ngx_file_io_uring_read(&c->file, c->buf->pos, c->body_start, 0,
r->pool);
if (n != NGX_AGAIN) {
c->reading = 0;
return n;
}
c->reading = 1;
c->file.io_uring->data = r;
c->file.io_uring->handler = ngx_http_cache_io_uring_event_handler;
ngx_add_timer(&c->file.io_uring->event, 60000);
r->main->blocked++;
r->aio = 1;
return NGX_AGAIN;
}
#endif
#if (NGX_HAVE_FILE_AIO)
if (clcf->aio == NGX_HTTP_AIO_ON && ngx_file_aio) {
@ -736,6 +765,56 @@ ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c)
}
#if (NGX_HAVE_IO_URING)
static void
ngx_http_cache_io_uring_event_handler(ngx_event_t *ev)
{
ngx_event_io_uring_t *io_uring;
ngx_connection_t *c;
ngx_http_request_t *r;
io_uring = ev->data;
r = io_uring->data;
c = r->connection;
ngx_http_set_log_request(c->log, r);
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http file cache io_uring: \"%V?%V\"",
&r->uri, &r->args);
if (ev->timedout) {
ngx_log_error(NGX_LOG_ALERT, c->log, 0,
"io_uring operation took too long");
ev->timedout = 0;
return;
}
if (ev->timer_set) {
ngx_del_timer(ev);
}
r->main->blocked--;
r->aio = 0;
if (r->main->terminated) {
/*
* trigger connection event handler if the request was
* terminated
*/
c->write->handler(c->write);
} else {
r->write_event_handler(r);
ngx_http_run_posted_requests(c);
}
}
#endif
#if (NGX_HAVE_FILE_AIO)
static void

6
src/os/unix/ngx_files.c
View file

@ -20,7 +20,11 @@ static ssize_t ngx_writev_file(ngx_file_t *file, ngx_iovec_t *vec,
off_t offset);
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING)
ngx_uint_t ngx_io_uring = 1;
#elif (NGX_HAVE_FILE_AIO)
ngx_uint_t ngx_file_aio = 1;

10
src/os/unix/ngx_files.h
View file

@ -375,7 +375,15 @@ off_t ngx_fs_available(u_char *name);
#define ngx_set_stderr_n "dup2(STDERR_FILENO)"
#if (NGX_HAVE_FILE_AIO)
#if (NGX_HAVE_IO_URING)
ngx_int_t ngx_file_io_uring_init(ngx_file_t *file, ngx_pool_t *pool);
ssize_t ngx_file_io_uring_read(ngx_file_t *file, u_char *buf, size_t size,
off_t offset, ngx_pool_t *pool);
extern ngx_uint_t ngx_io_uring;
#elif (NGX_HAVE_FILE_AIO)
ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool);
ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,

5
src/os/unix/ngx_linux_config.h
View file

@ -97,6 +97,11 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size);
#include <sys/eventfd.h>
#endif
#include <sys/syscall.h>
#if (NGX_HAVE_LIBURING)
#include <liburing.h>
#endif
#if (NGX_HAVE_FILE_AIO)
#include <linux/aio_abi.h>
typedef struct iocb ngx_aiocb_t;

165
src/os/unix/ngx_linux_io_uring_read.c Normal file
View file

@ -0,0 +1,165 @@
/*
* Copyright (C) Nginx, Inc.
*/
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_event.h>
extern struct io_uring *ngx_io_uring_ring;
static void ngx_file_io_uring_event_handler(ngx_event_t *ev);
ngx_int_t
ngx_file_io_uring_init(ngx_file_t *file, ngx_pool_t *pool)
{
ngx_event_io_uring_t *io_uring;
io_uring = ngx_pcalloc(pool, sizeof(ngx_event_io_uring_t));
if (io_uring == NULL) {
return NGX_ERROR;
}
io_uring->file = file;
io_uring->fd = file->fd;
io_uring->event.data = io_uring;
io_uring->event.ready = 1;
io_uring->event.log = file->log;
file->io_uring = io_uring;
return NGX_OK;
}
ssize_t
ngx_file_io_uring_read(ngx_file_t *file, u_char *buf, size_t size,
off_t offset, ngx_pool_t *pool)
{
int ret;
ngx_err_t err;
ngx_event_t *ev;
ngx_event_io_uring_t *io_uring;
struct io_uring_sqe *sqe;
if (!ngx_io_uring || ngx_io_uring_ring == NULL) {
return ngx_read_file(file, buf, size, offset);
}
if (file->io_uring == NULL
&& ngx_file_io_uring_init(file, pool) != NGX_OK)
{
return NGX_ERROR;
}
io_uring = file->io_uring;
ev = &io_uring->event;
if (!ev->ready) {
ngx_log_error(NGX_LOG_ALERT, file->log, 0,
"second io_uring post for \"%V\"", &file->name);
return NGX_AGAIN;
}
ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0,
"io_uring complete:%d @%O:%uz %V",
ev->complete, offset, size, &file->name);
if (ev->complete) {
ev->active = 0;
ev->complete = 0;
if (io_uring->res >= 0) {
ngx_set_errno(0);
return io_uring->res;
}
err = (ngx_err_t) -io_uring->res;
ngx_set_errno(err);
ngx_log_error(NGX_LOG_CRIT, file->log, err,
"io_uring read \"%s\" failed", file->name.data);
if (err == NGX_EINVAL || err == NGX_ENOSYS
|| err == NGX_EOPNOTSUPP)
{
ngx_io_uring = 0;
return ngx_read_file(file, buf, size, offset);
}
return NGX_ERROR;
}
sqe = io_uring_get_sqe(ngx_io_uring_ring);
if (sqe == NULL) {
(void) io_uring_submit(ngx_io_uring_ring);
sqe = io_uring_get_sqe(ngx_io_uring_ring);
}
if (sqe == NULL) {
ngx_log_error(NGX_LOG_ALERT, file->log, 0,
"io_uring_get_sqe() failed");
return ngx_read_file(file, buf, size, offset);
}
io_uring_prep_read(sqe, file->fd, buf, size, offset);
io_uring_sqe_set_data(sqe, ev);
ev->handler = ngx_file_io_uring_event_handler;
ret = io_uring_submit(ngx_io_uring_ring);
if (ret > 0) {
ev->active = 1;
ev->ready = 0;
ev->complete = 0;
return NGX_AGAIN;
}
if (ret < 0) {
err = (ngx_err_t) -ret;
if (err == NGX_EAGAIN) {
return ngx_read_file(file, buf, size, offset);
}
ngx_log_error(NGX_LOG_CRIT, file->log, err,
"io_uring_submit(\"%V\") failed", &file->name);
if (err == NGX_ENOSYS || err == NGX_EINVAL
|| err == NGX_EOPNOTSUPP)
{
ngx_io_uring = 0;
return ngx_read_file(file, buf, size, offset);
}
return NGX_ERROR;
}
ngx_log_error(NGX_LOG_ALERT, file->log, 0,
"io_uring_submit(\"%V\") submitted no requests",
&file->name);
return ngx_read_file(file, buf, size, offset);
}
static void
ngx_file_io_uring_event_handler(ngx_event_t *ev)
{
ngx_event_io_uring_t *io_uring;
io_uring = ev->data;
ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0,
"io_uring event handler fd:%d %V",
io_uring->fd, &io_uring->file->name);
io_uring->handler(ev);
}