From acdbe31456def4a86a1d6ca7ac6f2c9d30874402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E9=80=9A=E6=B4=B2?= Date: Tue, 3 Dec 2019 15:07:05 +0800 Subject: [PATCH] nginx_io_uring.patch: Fix -EFAULT errors --- nginx_io_uring.patch | 214 ++++++++++++++++++++++++------------------- 1 file changed, 118 insertions(+), 96 deletions(-) diff --git a/nginx_io_uring.patch b/nginx_io_uring.patch index 90275b3..18766bd 100644 --- a/nginx_io_uring.patch +++ b/nginx_io_uring.patch @@ -1,24 +1,25 @@ -From 5e463cc0c3161dec121a693b0c4e589694eb9682 Mon Sep 17 00:00:00 2001 -From: Carter Li -Date: Thu, 13 Jun 2019 20:23:58 +0800 +From da87b6048f6a66da7d23d77e4009be5090b5a995 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=E6=9D=8E=E9=80=9A=E6=B4=B2?= +Date: Tue, 3 Dec 2019 15:00:52 +0800 Subject: [PATCH] Add io_uring support --- auto/unix | 39 ++--- - src/event/modules/ngx_epoll_module.c | 222 +++++---------------------- - src/event/ngx_event.h | 1 - + src/core/ngx_open_file_cache.c | 6 +- + src/event/modules/ngx_epoll_module.c | 218 +++++---------------------- + src/event/ngx_event.h | 4 +- src/os/unix/ngx_linux_aio_read.c | 44 +++--- src/os/unix/ngx_linux_config.h | 3 +- - 5 files changed, 68 insertions(+), 241 deletions(-) + 6 files changed, 72 insertions(+), 242 deletions(-) diff --git a/auto/unix b/auto/unix -index 43d3b25..5b8ade8 100644 +index ff9697a..2e2f63a 100644 --- a/auto/unix +++ b/auto/unix @@ -532,44 +532,23 @@ if [ $NGX_FILE_AIO = YES ]; then - + if [ $ngx_found = no ]; then - + - ngx_feature="Linux AIO support" + ngx_feature="Linux io_uring support (liburing)" ngx_feature_name="NGX_HAVE_FILE_AIO" @@ -40,7 +41,7 @@ index 43d3b25..5b8ade8 100644 + if (ret < 0) return 1; + io_uring_queue_exit(&ring);" . auto/feature - + if [ $ngx_found = yes ]; then have=NGX_HAVE_EVENTFD . auto/have have=NGX_HAVE_SYS_EVENTFD_H . auto/have @@ -67,24 +68,43 @@ index 43d3b25..5b8ade8 100644 + CORE_LIBS="$CORE_LIBS -luring" fi fi - + @@ -577,7 +556,7 @@ if [ $NGX_FILE_AIO = YES ]; then cat << END - + $0: no supported file AIO was found -Currently file AIO is supported on FreeBSD 4.3+ and Linux 2.6.22+ only +Currently file AIO is supported on FreeBSD 4.3+ and Linux 5.1.0+ (requires liburing) only - + END exit 1 +diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c +index b23ee78..682d48c 100644 +--- a/src/core/ngx_open_file_cache.c ++++ b/src/core/ngx_open_file_cache.c +@@ -869,11 +869,11 @@ ngx_open_and_stat_file(ngx_str_t *name, ngx_open_file_info_t *of, + if (!of->log) { + + /* +- * Use non-blocking open() not to hang on FIFO files, etc. +- * This flag has no effect on a regular files. ++ * Differs from plain read, IORING_OP_READV with O_NONBLOCK ++ * will return -EAGAIN if the operation may block. + */ + +- fd = ngx_open_file_wrapper(name, of, NGX_FILE_RDONLY|NGX_FILE_NONBLOCK, ++ fd = ngx_open_file_wrapper(name, of, NGX_FILE_RDONLY, + NGX_FILE_OPEN, 0, log); + + } else { diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c -index 76aee08..812b43a 100644 +index 98e3ce7..2f0bd26 100644 --- a/src/event/modules/ngx_epoll_module.c +++ b/src/event/modules/ngx_epoll_module.c @@ -75,23 +75,6 @@ int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout) #define SYS_eventfd 323 #endif - + -#if (NGX_HAVE_FILE_AIO) - -#define SYS_io_setup 245 @@ -103,23 +123,23 @@ index 76aee08..812b43a 100644 - -#endif #endif /* NGX_TEST_BUILD_EPOLL */ - - + + @@ -124,7 +107,7 @@ static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); - + #if (NGX_HAVE_FILE_AIO) -static void ngx_epoll_eventfd_handler(ngx_event_t *ev); +static void ngx_epoll_io_uring_handler(ngx_event_t *ev); #endif - + static void *ngx_epoll_create_conf(ngx_cycle_t *cycle); @@ -141,13 +124,10 @@ static ngx_connection_t notify_conn; #endif - + #if (NGX_HAVE_FILE_AIO) +struct io_uring ngx_ring; - + -int ngx_eventfd = -1; -aio_context_t ngx_aio_ctx = 0; - @@ -129,12 +149,12 @@ index 76aee08..812b43a 100644 +static ngx_event_t ngx_ring_event; +static ngx_connection_t ngx_ring_conn; #endif - + #if (NGX_HAVE_EPOLLRDHUP) @@ -217,102 +197,40 @@ ngx_module_t ngx_epoll_module = { - + #if (NGX_HAVE_FILE_AIO) - + -/* - * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly - * as syscalls instead of libaio usage, because the library header file @@ -168,7 +188,7 @@ index 76aee08..812b43a 100644 { - int n; struct epoll_event ee; - + -#if (NGX_HAVE_SYS_EVENTFD_H) - ngx_eventfd = eventfd(0, 0); -#else @@ -200,7 +220,7 @@ index 76aee08..812b43a 100644 + "io_uring_queue_init() failed"); goto failed; } - + - ngx_eventfd_event.data = &ngx_eventfd_conn; - ngx_eventfd_event.handler = ngx_epoll_eventfd_handler; - ngx_eventfd_event.log = cycle->log; @@ -215,27 +235,27 @@ index 76aee08..812b43a 100644 + ngx_ring_conn.fd = ngx_ring.ring_fd; + ngx_ring_conn.read = &ngx_ring_event; + ngx_ring_conn.log = cycle->log; - + ee.events = EPOLLIN|EPOLLET; - ee.data.ptr = &ngx_eventfd_conn; + ee.data.ptr = &ngx_ring_conn; - + - if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) != -1) { + if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_ring.ring_fd, &ee) != -1) { return; } - + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed"); - + - if (io_destroy(ngx_aio_ctx) == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "io_destroy() failed"); - } + io_uring_queue_exit(&ngx_ring); - + failed: - + - if (close(ngx_eventfd) == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "eventfd close() failed"); @@ -246,11 +266,11 @@ index 76aee08..812b43a 100644 + ngx_ring.ring_fd = 0; ngx_file_aio = 0; } - + @@ -549,23 +467,11 @@ ngx_epoll_done(ngx_cycle_t *cycle) - + #if (NGX_HAVE_FILE_AIO) - + - if (ngx_eventfd != -1) { - - if (io_destroy(ngx_aio_ctx) == -1) { @@ -268,15 +288,15 @@ index 76aee08..812b43a 100644 + io_uring_queue_exit(&ngx_ring); + ngx_ring.ring_fd = 0; } - + - ngx_aio_ctx = 0; - #endif - + ngx_free(event_list); -@@ -940,83 +846,31 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) +@@ -939,83 +845,31 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) #if (NGX_HAVE_FILE_AIO) - + static void -ngx_epoll_eventfd_handler(ngx_event_t *ev) +ngx_epoll_io_uring_handler(ngx_event_t *ev) @@ -294,7 +314,7 @@ index 76aee08..812b43a 100644 - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler"); - - n = read(ngx_eventfd, &ready, 8); -- + - err = ngx_errno; - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n); @@ -308,7 +328,9 @@ index 76aee08..812b43a 100644 - ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed"); - return; - } - ++ ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0, ++ "io_uring_peek_cqe: START"); + - ngx_log_error(NGX_LOG_ALERT, ev->log, 0, - "read(eventfd) returned only %d bytes", n); - return; @@ -339,81 +361,80 @@ index 76aee08..812b43a 100644 - e->complete = 1; - e->active = 0; - e->ready = 1; -- -- aio = e->data; -- aio->res = event[i].res; -- -- ngx_post_event(e, &ngx_posted_events); -- } -+ ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0, -+ "io_uring_peek_cqe: START"); -+ + while (io_uring_peek_cqe(&ngx_ring, &cqe) >= 0 && cqe) { + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_event: %p %d %d", + cqe->user_data, cqe->res, cqe->flags); - -- continue; -- } + +- aio = e->data; +- aio->res = event[i].res; + e = (ngx_event_t *) io_uring_cqe_get_data(cqe); + e->complete = 1; + e->active = 0; + e->ready = 1; - + +- ngx_post_event(e, &ngx_posted_events); +- } ++ aio = e->data; ++ aio->res = cqe->res; + +- continue; +- } ++ io_uring_cqe_seen(&ngx_ring, cqe); + - if (events == 0) { - return; - } -+ aio = e->data; -+ aio->res = cqe->res; - +- - /* events == -1 */ - ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, - "io_getevents() failed"); - return; -+ io_uring_cqe_seen(&ngx_ring, cqe); -+ + ngx_post_event(e, &ngx_posted_events); } } - + diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h -index bb77c4a..fcb0812 100644 +index 97f9673..eb17346 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h -@@ -167,7 +167,6 @@ struct ngx_event_aio_s { +@@ -160,7 +160,9 @@ struct ngx_event_aio_s { size_t nbytes; #endif - + - ngx_aiocb_t aiocb; ++ /* Make sure that this iov has the same lifecycle with its associated aio event */ ++ struct iovec iov; ++ ngx_event_t event; }; - + diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c -index 9f0a6c1..8826531 100644 +index 9f0a6c1..52849e3 100644 --- a/src/os/unix/ngx_linux_aio_read.c +++ b/src/os/unix/ngx_linux_aio_read.c @@ -10,19 +10,12 @@ #include - - + + -extern int ngx_eventfd; -extern aio_context_t ngx_aio_ctx; +extern struct io_uring ngx_ring; - - + + static void ngx_file_aio_event_handler(ngx_event_t *ev); - - + + -static int -io_submit(aio_context_t ctx, long n, struct iocb **paiocb) -{ - return syscall(SYS_io_submit, ctx, n, paiocb); -} - - + ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) -@@ -50,10 +43,11 @@ ssize_t +@@ -50,10 +43,10 @@ ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, ngx_pool_t *pool) { @@ -425,24 +446,16 @@ index 9f0a6c1..8826531 100644 + ngx_event_t *ev; + ngx_event_aio_t *aio; + struct io_uring_sqe *sqe; -+ struct iovec iov; - + if (!ngx_file_aio) { return ngx_read_file(file, buf, size, offset); -@@ -93,22 +87,24 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, +@@ -93,22 +86,25 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, return NGX_ERROR; } - + - ngx_memzero(&aio->aiocb, sizeof(struct iocb)); + sqe = io_uring_get_sqe(&ngx_ring); -+ -+ if (!sqe) { -+ ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, -+ "aio no sqe left:%d @%O:%uz %V", -+ ev->complete, offset, size, &file->name); -+ return ngx_read_file(file, buf, size, offset); -+ } - + - aio->aiocb.aio_data = (uint64_t) (uintptr_t) ev; - aio->aiocb.aio_lio_opcode = IOCB_CMD_PREAD; - aio->aiocb.aio_fildes = file->fd; @@ -451,17 +464,25 @@ index 9f0a6c1..8826531 100644 - aio->aiocb.aio_offset = offset; - aio->aiocb.aio_flags = IOCB_FLAG_RESFD; - aio->aiocb.aio_resfd = ngx_eventfd; -+ iov.iov_base = buf; -+ iov.iov_len = size; -+ io_uring_prep_readv(sqe, file->fd, &iov, 1, offset); -+ io_uring_sqe_set_data(sqe, ev); - ++ if (!sqe) { ++ ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, ++ "aio no sqe left:%d @%O:%uz %V", ++ ev->complete, offset, size, &file->name); ++ return ngx_read_file(file, buf, size, offset); ++ } + - ev->handler = ngx_file_aio_event_handler; - ++ /* We must store iov into heap to prevent kernel from returning -EFAULT */ ++ aio->iov.iov_base = buf; ++ aio->iov.iov_len = size; ++ io_uring_prep_readv(sqe, file->fd, &aio->iov, 1, offset); ++ io_uring_sqe_set_data(sqe, ev); + - piocb[0] = &aio->aiocb; -+ ev->handler = ngx_file_aio_event_handler; - + - if (io_submit(ngx_aio_ctx, 1, piocb) == 1) { ++ ev->handler = ngx_file_aio_event_handler; ++ + if (io_uring_submit(&ngx_ring) == 1) { ev->active = 1; ev->ready = 0; @@ -478,7 +499,8 @@ index 3036cae..daf5b4c 100644 -typedef struct iocb ngx_aiocb_t; +#include #endif + + +-- +2.24.0 - --- -2.22.0