diff --git a/nginx_io_uring.patch b/nginx_io_uring.patch index 18766bd..64a39be 100644 --- a/nginx_io_uring.patch +++ b/nginx_io_uring.patch @@ -1,16 +1,16 @@ -From da87b6048f6a66da7d23d77e4009be5090b5a995 Mon Sep 17 00:00:00 2001 +From a052b2d1adf583233454a6d7f89963544714ff58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E9=80=9A=E6=B4=B2?= -Date: Tue, 3 Dec 2019 15:00:52 +0800 +Date: Fri, 14 Aug 2020 10:44:44 +0800 Subject: [PATCH] Add io_uring support --- auto/unix | 39 ++--- src/core/ngx_open_file_cache.c | 6 +- - src/event/modules/ngx_epoll_module.c | 218 +++++---------------------- + src/event/modules/ngx_epoll_module.c | 225 ++++++--------------------- src/event/ngx_event.h | 4 +- - src/os/unix/ngx_linux_aio_read.c | 44 +++--- - src/os/unix/ngx_linux_config.h | 3 +- - 6 files changed, 72 insertions(+), 242 deletions(-) + src/os/unix/ngx_linux_aio_read.c | 63 +++++--- + src/os/unix/ngx_linux_config.h | 4 - + 6 files changed, 98 insertions(+), 243 deletions(-) diff --git a/auto/unix b/auto/unix index ff9697a..2e2f63a 100644 @@ -98,10 +98,21 @@ index b23ee78..682d48c 100644 } else { diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c -index 98e3ce7..2f0bd26 100644 +index 98e3ce7..09dda1d 100644 --- a/src/event/modules/ngx_epoll_module.c +++ b/src/event/modules/ngx_epoll_module.c -@@ -75,23 +75,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout) +@@ -9,6 +9,10 @@ + #include + #include + ++#if (NGX_HAVE_FILE_AIO) ++#include ++#endif ++ + + #if (NGX_TEST_BUILD_EPOLL) + +@@ -75,23 +79,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout) #define SYS_eventfd 323 #endif @@ -125,7 +136,7 @@ index 98e3ce7..2f0bd26 100644 #endif /* NGX_TEST_BUILD_EPOLL */ -@@ -124,7 +107,7 @@ static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, +@@ -124,7 +111,7 @@ static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); #if (NGX_HAVE_FILE_AIO) @@ -134,11 +145,12 @@ index 98e3ce7..2f0bd26 100644 #endif static void *ngx_epoll_create_conf(ngx_cycle_t *cycle); -@@ -141,13 +124,10 @@ static ngx_connection_t notify_conn; +@@ -141,13 +128,11 @@ static ngx_connection_t notify_conn; #endif #if (NGX_HAVE_FILE_AIO) +struct io_uring ngx_ring; ++struct io_uring_params ngx_ring_params; -int ngx_eventfd = -1; -aio_context_t ngx_aio_ctx = 0; @@ -151,7 +163,7 @@ index 98e3ce7..2f0bd26 100644 #endif #if (NGX_HAVE_EPOLLRDHUP) -@@ -217,102 +197,40 @@ ngx_module_t ngx_epoll_module = { +@@ -217,102 +202,40 @@ ngx_module_t ngx_epoll_module = { #if (NGX_HAVE_FILE_AIO) @@ -214,10 +226,10 @@ index 98e3ce7..2f0bd26 100644 - } - - if (io_setup(epcf->aio_requests, &ngx_aio_ctx) == -1) { -+ if (io_uring_queue_init(64, &ngx_ring, 0) < 0) { ++ if (io_uring_queue_init_params(64, &ngx_ring, &ngx_ring_params) < 0) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "io_setup() failed"); -+ "io_uring_queue_init() failed"); ++ "io_uring_queue_init_params() failed"); goto failed; } @@ -267,7 +279,7 @@ index 98e3ce7..2f0bd26 100644 ngx_file_aio = 0; } -@@ -549,23 +467,11 @@ ngx_epoll_done(ngx_cycle_t *cycle) +@@ -549,23 +472,11 @@ ngx_epoll_done(ngx_cycle_t *cycle) #if (NGX_HAVE_FILE_AIO) @@ -294,7 +306,7 @@ index 98e3ce7..2f0bd26 100644 #endif ngx_free(event_list); -@@ -939,83 +845,31 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) +@@ -939,84 +850,36 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) #if (NGX_HAVE_FILE_AIO) static void @@ -307,35 +319,49 @@ index 98e3ce7..2f0bd26 100644 - ngx_err_t err; ngx_event_t *e; + struct io_uring_cqe *cqe; ++ unsigned head; ++ unsigned cqe_count = 0; ngx_event_aio_t *aio; - struct io_event event[64]; - struct timespec ts; -- + - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler"); -- ++ ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0, ++ "io_uring_peek_cqe: START"); + - n = read(ngx_eventfd, &ready, 8); ++ io_uring_for_each_cqe(&ngx_ring, head, cqe) { ++ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, ++ "io_event: %p %d %d", ++ cqe->user_data, cqe->res, cqe->flags); - err = ngx_errno; -- ++ e = (ngx_event_t *) io_uring_cqe_get_data(cqe); ++ e->complete = 1; ++ e->active = 0; ++ e->ready = 1; + - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n); -- ++ aio = e->data; ++ aio->res = cqe->res; + - if (n != 8) { - if (n == -1) { - if (err == NGX_EAGAIN) { - return; - } -- ++ ++cqe_count; + - ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed"); - return; - } -+ ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0, -+ "io_uring_peek_cqe: START"); - +- - ngx_log_error(NGX_LOG_ALERT, ev->log, 0, - "read(eventfd) returned only %d bytes", n); - return; -- } -- ++ ngx_post_event(e, &ngx_posted_events); + } + - ts.tv_sec = 0; - ts.tv_nsec = 0; - @@ -361,27 +387,16 @@ index 98e3ce7..2f0bd26 100644 - e->complete = 1; - e->active = 0; - e->ready = 1; -+ while (io_uring_peek_cqe(&ngx_ring, &cqe) >= 0 && cqe) { -+ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, -+ "io_event: %p %d %d", -+ cqe->user_data, cqe->res, cqe->flags); - +- - aio = e->data; - aio->res = event[i].res; -+ e = (ngx_event_t *) io_uring_cqe_get_data(cqe); -+ e->complete = 1; -+ e->active = 0; -+ e->ready = 1; - +- - ngx_post_event(e, &ngx_posted_events); - } -+ aio = e->data; -+ aio->res = cqe->res; - +- - continue; - } -+ io_uring_cqe_seen(&ngx_ring, cqe); - +- - if (events == 0) { - return; - } @@ -390,10 +405,11 @@ index 98e3ce7..2f0bd26 100644 - ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, - "io_getevents() failed"); - return; -+ ngx_post_event(e, &ngx_posted_events); - } +- } ++ io_uring_cq_advance(&ngx_ring, cqe_count); } + #endif diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index 97f9673..eb17346 100644 --- a/src/event/ngx_event.h @@ -410,31 +426,34 @@ index 97f9673..eb17346 100644 }; diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c -index 9f0a6c1..52849e3 100644 +index 9f0a6c1..033bd56 100644 --- a/src/os/unix/ngx_linux_aio_read.c +++ b/src/os/unix/ngx_linux_aio_read.c -@@ -10,19 +10,12 @@ +@@ -9,19 +9,15 @@ + #include #include ++#include -extern int ngx_eventfd; -extern aio_context_t ngx_aio_ctx; -+extern struct io_uring ngx_ring; ++extern struct io_uring ngx_ring; ++extern struct io_uring_params ngx_ring_params; - static void ngx_file_aio_event_handler(ngx_event_t *ev); +-static void ngx_file_aio_event_handler(ngx_event_t *ev); ++static void ngx_file_aio_event_handler(ngx_event_t *ev); -static int -io_submit(aio_context_t ctx, long n, struct iocb **paiocb) -{ - return syscall(SYS_io_submit, ctx, n, paiocb); -} -- + ngx_int_t - ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) -@@ -50,10 +43,10 @@ ssize_t +@@ -50,10 +46,10 @@ ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, ngx_pool_t *pool) { @@ -449,7 +468,7 @@ index 9f0a6c1..52849e3 100644 if (!ngx_file_aio) { return ngx_read_file(file, buf, size, offset); -@@ -93,22 +86,25 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, +@@ -93,22 +89,41 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, return NGX_ERROR; } @@ -472,10 +491,26 @@ index 9f0a6c1..52849e3 100644 + } - ev->handler = ngx_file_aio_event_handler; -+ /* We must store iov into heap to prevent kernel from returning -EFAULT */ -+ aio->iov.iov_base = buf; -+ aio->iov.iov_len = size; -+ io_uring_prep_readv(sqe, file->fd, &aio->iov, 1, offset); ++ if (__builtin_expect(!!(ngx_ring_params.features & IORING_FEAT_CUR_PERSONALITY), 1)) { ++ /* ++ * `io_uring_prep_read` is faster than `io_uring_prep_readv`, because the kernel ++ * doesn't need to import iovecs in advance. ++ * ++ * If the kernel supports `IORING_FEAT_CUR_PERSONALITY`, it should support ++ * non-vectored read/write commands too. ++ * ++ * It's not perfect, but avoids an extra feature-test syscall. ++ */ ++ io_uring_prep_read(sqe, file->fd, buf, size, offset); ++ } else { ++ /* ++ * We must store iov into heap to prevent kernel from returning -EFAULT ++ * in case `IORING_FEAT_SUBMIT_STABLE` is not supported ++ */ ++ aio->iov.iov_base = buf; ++ aio->iov.iov_len = size; ++ io_uring_prep_readv(sqe, file->fd, &aio->iov, 1, offset); ++ } + io_uring_sqe_set_data(sqe, ev); - piocb[0] = &aio->aiocb; @@ -488,19 +523,20 @@ index 9f0a6c1..52849e3 100644 ev->ready = 0; ev->complete = 0; diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h -index 3036cae..daf5b4c 100644 +index 3036cae..0e461dd 100644 --- a/src/os/unix/ngx_linux_config.h +++ b/src/os/unix/ngx_linux_config.h -@@ -94,8 +94,7 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size); +@@ -93,10 +93,6 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size); + #include #endif #include - #if (NGX_HAVE_FILE_AIO) +-#if (NGX_HAVE_FILE_AIO) -#include -typedef struct iocb ngx_aiocb_t; -+#include - #endif +-#endif + #if (NGX_HAVE_CAPABILITIES) -- -2.24.0 +2.25.1