diff options
| author | thegeorg <thegeorg@yandex-team.com> | 2023-07-26 17:26:21 +0300 |
|---|---|---|
| committer | thegeorg <thegeorg@yandex-team.com> | 2023-07-26 17:26:21 +0300 |
| commit | 3785d5f97965bccf048718d8717904cf50f9f8f9 (patch) | |
| tree | b7ce8ae67d7eb7fcf7767c54379f0564c281147f /contrib | |
| parent | 1f6b57071583f89299bb5abd3863d594f23c5be5 (diff) | |
| download | ydb-3785d5f97965bccf048718d8717904cf50f9f8f9.tar.gz | |
Update contrib/libs/liburing to 2.4
Diffstat (limited to 'contrib')
124 files changed, 4706 insertions, 1128 deletions
diff --git a/contrib/libs/liburing/CHANGELOG b/contrib/libs/liburing/CHANGELOG index 09511af427..71ca3919e1 100644 --- a/contrib/libs/liburing/CHANGELOG +++ b/contrib/libs/liburing/CHANGELOG @@ -1,3 +1,21 @@ +liburing-2.4 release + +- Add io_uring_{major,minor,check}_version() functions. +- Add IO_URING_{MAJOR,MINOR,CHECK}_VERSION() macros. +- FFI support (for non-C/C++ languages integration). +- Add io_uring_prep_msg_ring_cqe_flags() function. +- Deprecate --nolibc configure option. +- CONFIG_NOLIBC is always enabled on x86-64, x86, and aarch64. +- Add support for IORING_REGISTER_USE_REGISTERED_RING and use if available. +- Add io_uring_close_ring_fd() function. +- Add io_uring_prep_msg_ring_fd_alloc function. +- Add io_uring_free_buf_ring() and io_uring_setup_buf_ring() functions. +- Ensure that io_uring_prep_accept_direct(), io_uring_prep_openat_direct(), + io_uring_prep_openat2_direct(), io_uring_prep_msg_ring_fd(), and + io_uring_prep_socket_direct() factor in being called with + IORING_FILE_INDEX_ALLOC for allocating a direct descriptor. +- Add io_uring_prep_sendto() function. + liburing-2.3 release - Support non-libc build for aarch64. diff --git a/contrib/libs/liburing/CMakeLists.linux-aarch64.txt b/contrib/libs/liburing/CMakeLists.linux-aarch64.txt index 63d62ff3ed..efada53e84 100644 --- a/contrib/libs/liburing/CMakeLists.linux-aarch64.txt +++ b/contrib/libs/liburing/CMakeLists.linux-aarch64.txt @@ -19,8 +19,10 @@ target_link_libraries(contrib-libs-liburing PUBLIC contrib-libs-linux-headers ) target_sources(contrib-libs-liburing PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/nolibc.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/queue.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/register.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/setup.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/syscall.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/version.c ) diff --git a/contrib/libs/liburing/CMakeLists.linux-x86_64.txt b/contrib/libs/liburing/CMakeLists.linux-x86_64.txt index 63d62ff3ed..efada53e84 100644 --- a/contrib/libs/liburing/CMakeLists.linux-x86_64.txt +++ b/contrib/libs/liburing/CMakeLists.linux-x86_64.txt @@ -19,8 +19,10 @@ target_link_libraries(contrib-libs-liburing PUBLIC contrib-libs-linux-headers ) target_sources(contrib-libs-liburing PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/nolibc.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/queue.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/register.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/setup.c ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/syscall.c + ${CMAKE_SOURCE_DIR}/contrib/libs/liburing/src/version.c ) diff --git a/contrib/libs/liburing/README b/contrib/libs/liburing/README index 80d2b3dc5d..9c881ae757 100644 --- a/contrib/libs/liburing/README +++ b/contrib/libs/liburing/README @@ -47,6 +47,54 @@ the kernel io_uring support. Please note that this suite isn't expected to pass on older kernels, and may even crash or hang older kernels! +Building liburing +----------------- + + # + # Prepare build config (optional). + # + # --cc specifies the C compiler. + # --cxx speficies the C++ compiler. + # + ./configure --cc=gcc --cxx=g++; + + # + # Build liburing. + # + make -j$(nproc); + + # + # Install liburing (headers, shared/static libs, and manpage). + # + sudo make install; + +See './configure --help' for more information about build config options. + + +FFI support +----------- + +By default, the build results in 4 lib files: + + 2 shared libs: + + liburing.so + liburing-ffi.so + + 2 static libs: + + liburing.a + liburing-ffi.a + +Languages and applications that can't use 'static inline' functions in +liburing.h should use the FFI variants. + +liburing's main public interface lives in liburing.h as 'static inline' +functions. Users wishing to consume liburing purely as a binary dependency +should link against liburing-ffi. It contains definitions for every 'static +inline' function. + + License ------- diff --git a/contrib/libs/liburing/config-host.h b/contrib/libs/liburing/config-host.h index 1488c383ea..2f1d299bd5 100644 --- a/contrib/libs/liburing/config-host.h +++ b/contrib/libs/liburing/config-host.h @@ -2,6 +2,7 @@ * Automatically generated by configure - do not modify * Configured with: * './configure' * '--prefix=/var/empty/liburing' * '--includedir=/var/empty/tmp/out/include' * '--mandir=/var/empty/tmp/out/share/man' */ +#define CONFIG_NOLIBC #define CONFIG_HAVE_KERNEL_RWF_T #define CONFIG_HAVE_KERNEL_TIMESPEC #define CONFIG_HAVE_OPEN_HOW @@ -11,3 +12,5 @@ #define CONFIG_HAVE_UCONTEXT #define CONFIG_HAVE_STRINGOP_OVERFLOW #define CONFIG_HAVE_ARRAY_BOUNDS +#define CONFIG_HAVE_NVME_URING +#define CONFIG_HAVE_FANOTIFY diff --git a/contrib/libs/liburing/src/arch/aarch64/lib.h b/contrib/libs/liburing/src/arch/aarch64/lib.h index 5a75c1a672..3b701b1fb7 100644 --- a/contrib/libs/liburing/src/arch/aarch64/lib.h +++ b/contrib/libs/liburing/src/arch/aarch64/lib.h @@ -21,7 +21,7 @@ static inline long __get_page_size(void) ssize_t x; x = __sys_read(fd, buf, sizeof(buf)); - if (x < sizeof(buf)) + if (x < (long) sizeof(buf)) break; if (buf[0] == AT_PAGESZ) { diff --git a/contrib/libs/liburing/src/include/liburing.h b/contrib/libs/liburing/src/include/liburing.h index 12a703fdd2..e6dc9a792f 100644 --- a/contrib/libs/liburing/src/include/liburing.h +++ b/contrib/libs/liburing/src/include/liburing.h @@ -23,6 +23,7 @@ #include <linux/swab.h> #include "liburing/compat.h" #include "liburing/io_uring.h" +#include "liburing/io_uring_version.h" #include "liburing/barrier.h" #ifndef uring_unlikely @@ -33,6 +34,10 @@ #define uring_likely(cond) __builtin_expect(!!(cond), 1) #endif +#ifndef IOURINGINLINE +#define IOURINGINLINE static inline +#endif + #ifdef __alpha__ /* * alpha and mips are the exceptions, all other architectures have @@ -151,7 +156,7 @@ struct io_uring_probe *io_uring_get_probe(void); */ void io_uring_free_probe(struct io_uring_probe *probe); -static inline int io_uring_opcode_supported(const struct io_uring_probe *p, +IOURINGINLINE int io_uring_opcode_supported(const struct io_uring_probe *p, int op) { if (op > p->last_op) @@ -226,6 +231,7 @@ int io_uring_register_iowq_max_workers(struct io_uring *ring, unsigned int *values); int io_uring_register_ring_fd(struct io_uring *ring); int io_uring_unregister_ring_fd(struct io_uring *ring); +int io_uring_close_ring_fd(struct io_uring *ring); int io_uring_register_buf_ring(struct io_uring *ring, struct io_uring_buf_reg *reg, unsigned int flags); int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid); @@ -251,6 +257,16 @@ int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg, unsigned int nr_args); /* + * Mapped buffer ring alloc/register + unregister/free helpers + */ +struct io_uring_buf_ring *io_uring_setup_buf_ring(struct io_uring *ring, + unsigned int nentries, + int bgid, unsigned int flags, + int *ret); +int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br, + unsigned int nentries, int bgid); + +/* * Helper for the peek/wait single cqe functions. Exported because of that, * but probably shouldn't be used directly in an application. */ @@ -283,8 +299,7 @@ int __io_uring_get_cqe(struct io_uring *ring, /* * Must be called after io_uring_for_each_cqe() */ -static inline void io_uring_cq_advance(struct io_uring *ring, - unsigned nr) +IOURINGINLINE void io_uring_cq_advance(struct io_uring *ring, unsigned nr) { if (nr) { struct io_uring_cq *cq = &ring->cq; @@ -301,7 +316,7 @@ static inline void io_uring_cq_advance(struct io_uring *ring, * Must be called after io_uring_{peek,wait}_cqe() after the cqe has * been processed by the application. */ -static inline void io_uring_cqe_seen(struct io_uring *ring, +IOURINGINLINE void io_uring_cqe_seen(struct io_uring *ring, struct io_uring_cqe *cqe) { if (cqe) @@ -316,12 +331,12 @@ static inline void io_uring_cqe_seen(struct io_uring *ring, * Associate pointer @data with the sqe, for later retrieval from the cqe * at command completion time with io_uring_cqe_get_data(). */ -static inline void io_uring_sqe_set_data(struct io_uring_sqe *sqe, void *data) +IOURINGINLINE void io_uring_sqe_set_data(struct io_uring_sqe *sqe, void *data) { sqe->user_data = (unsigned long) data; } -static inline void *io_uring_cqe_get_data(const struct io_uring_cqe *cqe) +IOURINGINLINE void *io_uring_cqe_get_data(const struct io_uring_cqe *cqe) { return (void *) (uintptr_t) cqe->user_data; } @@ -331,13 +346,13 @@ static inline void *io_uring_cqe_get_data(const struct io_uring_cqe *cqe) * time with io_uring_cqe_get_data64. Just like the non-64 variants, except * these store a 64-bit type rather than a data pointer. */ -static inline void io_uring_sqe_set_data64(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_sqe_set_data64(struct io_uring_sqe *sqe, __u64 data) { sqe->user_data = data; } -static inline __u64 io_uring_cqe_get_data64(const struct io_uring_cqe *cqe) +IOURINGINLINE __u64 io_uring_cqe_get_data64(const struct io_uring_cqe *cqe) { return cqe->user_data; } @@ -347,20 +362,20 @@ static inline __u64 io_uring_cqe_get_data64(const struct io_uring_cqe *cqe) */ #define LIBURING_HAVE_DATA64 -static inline void io_uring_sqe_set_flags(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_sqe_set_flags(struct io_uring_sqe *sqe, unsigned flags) { sqe->flags = (__u8) flags; } -static inline void __io_uring_set_target_fixed_file(struct io_uring_sqe *sqe, +IOURINGINLINE void __io_uring_set_target_fixed_file(struct io_uring_sqe *sqe, unsigned int file_index) { /* 0 means no fixed files, indexes should be encoded as "index + 1" */ sqe->file_index = file_index + 1; } -static inline void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd, const void *addr, unsigned len, __u64 offset) { @@ -379,29 +394,28 @@ static inline void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd, sqe->__pad2[0] = 0; } -/** - * @pre Either fd_in or fd_out must be a pipe. - * @param off_in If fd_in refers to a pipe, off_in must be (int64_t) -1; - * If fd_in does not refer to a pipe and off_in is (int64_t) -1, - * then bytes are read from fd_in starting from the file offset - * and it is adjust appropriately; - * If fd_in does not refer to a pipe and off_in is not - * (int64_t) -1, then the starting offset of fd_in will be - * off_in. - * @param off_out The description of off_in also applied to off_out. - * @param splice_flags see man splice(2) for description of flags. +/* + * io_uring_prep_splice() - Either @fd_in or @fd_out must be a pipe. + * + * - If @fd_in refers to a pipe, @off_in is ignored and must be set to -1. + * + * - If @fd_in does not refer to a pipe and @off_in is -1, then @nbytes are read + * from @fd_in starting from the file offset, which is incremented by the + * number of bytes read. + * + * - If @fd_in does not refer to a pipe and @off_in is not -1, then the starting + * offset of @fd_in will be @off_in. * * This splice operation can be used to implement sendfile by splicing to an * intermediate pipe first, then splice to the final destination. * In fact, the implementation of sendfile in kernel uses splice internally. * * NOTE that even if fd_in or fd_out refers to a pipe, the splice operation - * can still failed with EINVAL if one of the fd doesn't explicitly support - * splice operation, e.g. reading from terminal is unsupported from kernel 5.7 - * to 5.11. + * can still fail with EINVAL if one of the fd doesn't explicitly support splice + * operation, e.g. reading from terminal is unsupported from kernel 5.7 to 5.11. * Check issue #291 for more information. */ -static inline void io_uring_prep_splice(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_splice(struct io_uring_sqe *sqe, int fd_in, int64_t off_in, int fd_out, int64_t off_out, unsigned int nbytes, @@ -414,7 +428,7 @@ static inline void io_uring_prep_splice(struct io_uring_sqe *sqe, sqe->splice_flags = splice_flags; } -static inline void io_uring_prep_tee(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_tee(struct io_uring_sqe *sqe, int fd_in, int fd_out, unsigned int nbytes, unsigned int splice_flags) @@ -425,14 +439,14 @@ static inline void io_uring_prep_tee(struct io_uring_sqe *sqe, sqe->splice_flags = splice_flags; } -static inline void io_uring_prep_readv(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_readv(struct io_uring_sqe *sqe, int fd, const struct iovec *iovecs, unsigned nr_vecs, __u64 offset) { io_uring_prep_rw(IORING_OP_READV, sqe, fd, iovecs, nr_vecs, offset); } -static inline void io_uring_prep_readv2(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_readv2(struct io_uring_sqe *sqe, int fd, const struct iovec *iovecs, unsigned nr_vecs, __u64 offset, int flags) @@ -441,7 +455,7 @@ static inline void io_uring_prep_readv2(struct io_uring_sqe *sqe, int fd, sqe->rw_flags = flags; } -static inline void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd, void *buf, unsigned nbytes, __u64 offset, int buf_index) { @@ -449,14 +463,14 @@ static inline void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd, sqe->buf_index = (__u16) buf_index; } -static inline void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd, const struct iovec *iovecs, unsigned nr_vecs, __u64 offset) { io_uring_prep_rw(IORING_OP_WRITEV, sqe, fd, iovecs, nr_vecs, offset); } -static inline void io_uring_prep_writev2(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_writev2(struct io_uring_sqe *sqe, int fd, const struct iovec *iovecs, unsigned nr_vecs, __u64 offset, int flags) @@ -465,7 +479,7 @@ static inline void io_uring_prep_writev2(struct io_uring_sqe *sqe, int fd, sqe->rw_flags = flags; } -static inline void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, const void *buf, unsigned nbytes, __u64 offset, int buf_index) { @@ -473,21 +487,22 @@ static inline void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, sqe->buf_index = (__u16) buf_index; } -static inline void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd, struct msghdr *msg, unsigned flags) { io_uring_prep_rw(IORING_OP_RECVMSG, sqe, fd, msg, 1, 0); sqe->msg_flags = flags; } -static inline void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *sqe, int fd, - struct msghdr *msg, unsigned flags) +IOURINGINLINE void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *sqe, + int fd, struct msghdr *msg, + unsigned flags) { io_uring_prep_recvmsg(sqe, fd, msg, flags); sqe->ioprio |= IORING_RECV_MULTISHOT; } -static inline void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd, const struct msghdr *msg, unsigned flags) { @@ -495,7 +510,7 @@ static inline void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd, sqe->msg_flags = flags; } -static inline unsigned __io_uring_prep_poll_mask(unsigned poll_mask) +IOURINGINLINE unsigned __io_uring_prep_poll_mask(unsigned poll_mask) { #if __BYTE_ORDER == __BIG_ENDIAN poll_mask = __swahw32(poll_mask); @@ -503,28 +518,28 @@ static inline unsigned __io_uring_prep_poll_mask(unsigned poll_mask) return poll_mask; } -static inline void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd, unsigned poll_mask) { io_uring_prep_rw(IORING_OP_POLL_ADD, sqe, fd, NULL, 0, 0); sqe->poll32_events = __io_uring_prep_poll_mask(poll_mask); } -static inline void io_uring_prep_poll_multishot(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_poll_multishot(struct io_uring_sqe *sqe, int fd, unsigned poll_mask) { io_uring_prep_poll_add(sqe, fd, poll_mask); sqe->len = IORING_POLL_ADD_MULTI; } -static inline void io_uring_prep_poll_remove(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_poll_remove(struct io_uring_sqe *sqe, __u64 user_data) { io_uring_prep_rw(IORING_OP_POLL_REMOVE, sqe, -1, NULL, 0, 0); sqe->addr = user_data; } -static inline void io_uring_prep_poll_update(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_poll_update(struct io_uring_sqe *sqe, __u64 old_user_data, __u64 new_user_data, unsigned poll_mask, unsigned flags) @@ -535,19 +550,19 @@ static inline void io_uring_prep_poll_update(struct io_uring_sqe *sqe, sqe->poll32_events = __io_uring_prep_poll_mask(poll_mask); } -static inline void io_uring_prep_fsync(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_fsync(struct io_uring_sqe *sqe, int fd, unsigned fsync_flags) { io_uring_prep_rw(IORING_OP_FSYNC, sqe, fd, NULL, 0, 0); sqe->fsync_flags = fsync_flags; } -static inline void io_uring_prep_nop(struct io_uring_sqe *sqe) +IOURINGINLINE void io_uring_prep_nop(struct io_uring_sqe *sqe) { io_uring_prep_rw(IORING_OP_NOP, sqe, -1, NULL, 0, 0); } -static inline void io_uring_prep_timeout(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_timeout(struct io_uring_sqe *sqe, struct __kernel_timespec *ts, unsigned count, unsigned flags) { @@ -555,7 +570,7 @@ static inline void io_uring_prep_timeout(struct io_uring_sqe *sqe, sqe->timeout_flags = flags; } -static inline void io_uring_prep_timeout_remove(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_timeout_remove(struct io_uring_sqe *sqe, __u64 user_data, unsigned flags) { io_uring_prep_rw(IORING_OP_TIMEOUT_REMOVE, sqe, -1, NULL, 0, 0); @@ -563,7 +578,7 @@ static inline void io_uring_prep_timeout_remove(struct io_uring_sqe *sqe, sqe->timeout_flags = flags; } -static inline void io_uring_prep_timeout_update(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_timeout_update(struct io_uring_sqe *sqe, struct __kernel_timespec *ts, __u64 user_data, unsigned flags) { @@ -573,7 +588,7 @@ static inline void io_uring_prep_timeout_update(struct io_uring_sqe *sqe, sqe->timeout_flags = flags | IORING_TIMEOUT_UPDATE; } -static inline void io_uring_prep_accept(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_accept(struct io_uring_sqe *sqe, int fd, struct sockaddr *addr, socklen_t *addrlen, int flags) { @@ -583,16 +598,19 @@ static inline void io_uring_prep_accept(struct io_uring_sqe *sqe, int fd, } /* accept directly into the fixed file table */ -static inline void io_uring_prep_accept_direct(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_accept_direct(struct io_uring_sqe *sqe, int fd, struct sockaddr *addr, socklen_t *addrlen, int flags, unsigned int file_index) { io_uring_prep_accept(sqe, fd, addr, addrlen, flags); + /* offset by 1 for allocation */ + if (file_index == IORING_FILE_INDEX_ALLOC) + file_index--; __io_uring_set_target_fixed_file(sqe, file_index); } -static inline void io_uring_prep_multishot_accept(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_multishot_accept(struct io_uring_sqe *sqe, int fd, struct sockaddr *addr, socklen_t *addrlen, int flags) { @@ -601,7 +619,7 @@ static inline void io_uring_prep_multishot_accept(struct io_uring_sqe *sqe, } /* multishot accept directly into the fixed file table */ -static inline void io_uring_prep_multishot_accept_direct(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_multishot_accept_direct(struct io_uring_sqe *sqe, int fd, struct sockaddr *addr, socklen_t *addrlen, @@ -611,7 +629,7 @@ static inline void io_uring_prep_multishot_accept_direct(struct io_uring_sqe *sq __io_uring_set_target_fixed_file(sqe, IORING_FILE_INDEX_ALLOC - 1); } -static inline void io_uring_prep_cancel64(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_cancel64(struct io_uring_sqe *sqe, __u64 user_data, int flags) { io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, -1, NULL, 0, 0); @@ -619,20 +637,20 @@ static inline void io_uring_prep_cancel64(struct io_uring_sqe *sqe, sqe->cancel_flags = (__u32) flags; } -static inline void io_uring_prep_cancel(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_cancel(struct io_uring_sqe *sqe, void *user_data, int flags) { io_uring_prep_cancel64(sqe, (__u64) (uintptr_t) user_data, flags); } -static inline void io_uring_prep_cancel_fd(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_cancel_fd(struct io_uring_sqe *sqe, int fd, unsigned int flags) { io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, fd, NULL, 0, 0); sqe->cancel_flags = (__u32) flags | IORING_ASYNC_CANCEL_FD; } -static inline void io_uring_prep_link_timeout(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_link_timeout(struct io_uring_sqe *sqe, struct __kernel_timespec *ts, unsigned flags) { @@ -640,14 +658,14 @@ static inline void io_uring_prep_link_timeout(struct io_uring_sqe *sqe, sqe->timeout_flags = flags; } -static inline void io_uring_prep_connect(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_connect(struct io_uring_sqe *sqe, int fd, const struct sockaddr *addr, socklen_t addrlen) { io_uring_prep_rw(IORING_OP_CONNECT, sqe, fd, addr, 0, addrlen); } -static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_files_update(struct io_uring_sqe *sqe, int *fds, unsigned nr_fds, int offset) { @@ -655,15 +673,15 @@ static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe, (__u64) offset); } -static inline void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd, - int mode, off_t offset, off_t len) +IOURINGINLINE void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd, + int mode, __u64 offset, __u64 len) { io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd, 0, (unsigned int) mode, (__u64) offset); sqe->addr = (__u64) len; } -static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd, +IOURINGINLINE void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd, const char *path, int flags, mode_t mode) { @@ -672,34 +690,37 @@ static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd, } /* open directly into the fixed file table */ -static inline void io_uring_prep_openat_direct(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_openat_direct(struct io_uring_sqe *sqe, int dfd, const char *path, int flags, mode_t mode, unsigned file_index) { io_uring_prep_openat(sqe, dfd, path, flags, mode); + /* offset by 1 for allocation */ + if (file_index == IORING_FILE_INDEX_ALLOC) + file_index--; __io_uring_set_target_fixed_file(sqe, file_index); } -static inline void io_uring_prep_close(struct io_uring_sqe *sqe, int fd) +IOURINGINLINE void io_uring_prep_close(struct io_uring_sqe *sqe, int fd) { io_uring_prep_rw(IORING_OP_CLOSE, sqe, fd, NULL, 0, 0); } -static inline void io_uring_prep_close_direct(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_close_direct(struct io_uring_sqe *sqe, unsigned file_index) { io_uring_prep_close(sqe, 0); __io_uring_set_target_fixed_file(sqe, file_index); } -static inline void io_uring_prep_read(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_read(struct io_uring_sqe *sqe, int fd, void *buf, unsigned nbytes, __u64 offset) { io_uring_prep_rw(IORING_OP_READ, sqe, fd, buf, nbytes, offset); } -static inline void io_uring_prep_write(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_write(struct io_uring_sqe *sqe, int fd, const void *buf, unsigned nbytes, __u64 offset) { @@ -707,37 +728,54 @@ static inline void io_uring_prep_write(struct io_uring_sqe *sqe, int fd, } struct statx; -static inline void io_uring_prep_statx(struct io_uring_sqe *sqe, int dfd, - const char *path, int flags, unsigned mask, - struct statx *statxbuf) +IOURINGINLINE void io_uring_prep_statx(struct io_uring_sqe *sqe, int dfd, + const char *path, int flags, + unsigned mask, struct statx *statxbuf) { io_uring_prep_rw(IORING_OP_STATX, sqe, dfd, path, mask, (__u64) (unsigned long) statxbuf); sqe->statx_flags = (__u32) flags; } -static inline void io_uring_prep_fadvise(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_fadvise(struct io_uring_sqe *sqe, int fd, __u64 offset, off_t len, int advice) { io_uring_prep_rw(IORING_OP_FADVISE, sqe, fd, NULL, (__u32) len, offset); sqe->fadvise_advice = (__u32) advice; } -static inline void io_uring_prep_madvise(struct io_uring_sqe *sqe, void *addr, +IOURINGINLINE void io_uring_prep_madvise(struct io_uring_sqe *sqe, void *addr, off_t length, int advice) { io_uring_prep_rw(IORING_OP_MADVISE, sqe, -1, addr, (__u32) length, 0); sqe->fadvise_advice = (__u32) advice; } -static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd, +IOURINGINLINE void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd, const void *buf, size_t len, int flags) { io_uring_prep_rw(IORING_OP_SEND, sqe, sockfd, buf, (__u32) len, 0); sqe->msg_flags = (__u32) flags; } -static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd, +IOURINGINLINE void io_uring_prep_send_set_addr(struct io_uring_sqe *sqe, + const struct sockaddr *dest_addr, + __u16 addr_len) +{ + sqe->addr2 = (unsigned long)(const void *)dest_addr; + sqe->addr_len = addr_len; +} + +IOURINGINLINE void io_uring_prep_sendto(struct io_uring_sqe *sqe, int sockfd, + const void *buf, size_t len, int flags, + const struct sockaddr *addr, + socklen_t addrlen) +{ + io_uring_prep_send(sqe, sockfd, buf, len, flags); + io_uring_prep_send_set_addr(sqe, addr, addrlen); +} + +IOURINGINLINE void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd, const void *buf, size_t len, int flags, unsigned zc_flags) { @@ -746,7 +784,7 @@ static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd, sqe->ioprio = zc_flags; } -static inline void io_uring_prep_send_zc_fixed(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_send_zc_fixed(struct io_uring_sqe *sqe, int sockfd, const void *buf, size_t len, int flags, unsigned zc_flags, @@ -757,7 +795,7 @@ static inline void io_uring_prep_send_zc_fixed(struct io_uring_sqe *sqe, sqe->buf_index = buf_index; } -static inline void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd, const struct msghdr *msg, unsigned flags) { @@ -765,22 +803,14 @@ static inline void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd, sqe->opcode = IORING_OP_SENDMSG_ZC; } -static inline void io_uring_prep_send_set_addr(struct io_uring_sqe *sqe, - const struct sockaddr *dest_addr, - __u16 addr_len) -{ - sqe->addr2 = (unsigned long)(const void *)dest_addr; - sqe->addr_len = addr_len; -} - -static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd, +IOURINGINLINE void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd, void *buf, size_t len, int flags) { io_uring_prep_rw(IORING_OP_RECV, sqe, sockfd, buf, (__u32) len, 0); sqe->msg_flags = (__u32) flags; } -static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe, int sockfd, void *buf, size_t len, int flags) { @@ -788,7 +818,7 @@ static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe, sqe->ioprio |= IORING_RECV_MULTISHOT; } -static inline struct io_uring_recvmsg_out * +IOURINGINLINE struct io_uring_recvmsg_out * io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *msgh) { unsigned long header = msgh->msg_controllen + msgh->msg_namelen + @@ -798,12 +828,12 @@ io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *msgh) return (struct io_uring_recvmsg_out *)buf; } -static inline void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o) +IOURINGINLINE void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o) { return (void *) &o[1]; } -static inline struct cmsghdr * +IOURINGINLINE struct cmsghdr * io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh) { @@ -814,7 +844,7 @@ io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o, msgh->msg_namelen); } -static inline struct cmsghdr * +IOURINGINLINE struct cmsghdr * io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh, struct cmsghdr *cmsg) { @@ -835,14 +865,14 @@ io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msg return cmsg; } -static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o, +IOURINGINLINE void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o, struct msghdr *msgh) { return (void *)((unsigned char *)io_uring_recvmsg_name(o) + msgh->msg_namelen + msgh->msg_controllen); } -static inline unsigned int +IOURINGINLINE unsigned int io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o, int buf_len, struct msghdr *msgh) { @@ -853,7 +883,7 @@ io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o, return (unsigned int) (payload_end - payload_start); } -static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd, +IOURINGINLINE void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd, const char *path, struct open_how *how) { io_uring_prep_rw(IORING_OP_OPENAT2, sqe, dfd, path, sizeof(*how), @@ -861,17 +891,20 @@ static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd, } /* open directly into the fixed file table */ -static inline void io_uring_prep_openat2_direct(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_openat2_direct(struct io_uring_sqe *sqe, int dfd, const char *path, struct open_how *how, unsigned file_index) { io_uring_prep_openat2(sqe, dfd, path, how); + /* offset by 1 for allocation */ + if (file_index == IORING_FILE_INDEX_ALLOC) + file_index--; __io_uring_set_target_fixed_file(sqe, file_index); } struct epoll_event; -static inline void io_uring_prep_epoll_ctl(struct io_uring_sqe *sqe, int epfd, +IOURINGINLINE void io_uring_prep_epoll_ctl(struct io_uring_sqe *sqe, int epfd, int fd, int op, struct epoll_event *ev) { @@ -879,7 +912,7 @@ static inline void io_uring_prep_epoll_ctl(struct io_uring_sqe *sqe, int epfd, (__u32) op, (__u32) fd); } -static inline void io_uring_prep_provide_buffers(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_provide_buffers(struct io_uring_sqe *sqe, void *addr, int len, int nr, int bgid, int bid) { @@ -888,33 +921,33 @@ static inline void io_uring_prep_provide_buffers(struct io_uring_sqe *sqe, sqe->buf_group = (__u16) bgid; } -static inline void io_uring_prep_remove_buffers(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_remove_buffers(struct io_uring_sqe *sqe, int nr, int bgid) { io_uring_prep_rw(IORING_OP_REMOVE_BUFFERS, sqe, nr, NULL, 0, 0); sqe->buf_group = (__u16) bgid; } -static inline void io_uring_prep_shutdown(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_shutdown(struct io_uring_sqe *sqe, int fd, int how) { io_uring_prep_rw(IORING_OP_SHUTDOWN, sqe, fd, NULL, (__u32) how, 0); } -static inline void io_uring_prep_unlinkat(struct io_uring_sqe *sqe, int dfd, +IOURINGINLINE void io_uring_prep_unlinkat(struct io_uring_sqe *sqe, int dfd, const char *path, int flags) { io_uring_prep_rw(IORING_OP_UNLINKAT, sqe, dfd, path, 0, 0); sqe->unlink_flags = (__u32) flags; } -static inline void io_uring_prep_unlink(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_unlink(struct io_uring_sqe *sqe, const char *path, int flags) { io_uring_prep_unlinkat(sqe, AT_FDCWD, path, flags); } -static inline void io_uring_prep_renameat(struct io_uring_sqe *sqe, int olddfd, +IOURINGINLINE void io_uring_prep_renameat(struct io_uring_sqe *sqe, int olddfd, const char *oldpath, int newdfd, const char *newpath, unsigned int flags) { @@ -924,13 +957,14 @@ static inline void io_uring_prep_renameat(struct io_uring_sqe *sqe, int olddfd, sqe->rename_flags = (__u32) flags; } -static inline void io_uring_prep_rename(struct io_uring_sqe *sqe, - const char *oldpath, const char *newpath) +IOURINGINLINE void io_uring_prep_rename(struct io_uring_sqe *sqe, + const char *oldpath, + const char *newpath) { io_uring_prep_renameat(sqe, AT_FDCWD, oldpath, AT_FDCWD, newpath, 0); } -static inline void io_uring_prep_sync_file_range(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_sync_file_range(struct io_uring_sqe *sqe, int fd, unsigned len, __u64 offset, int flags) { @@ -938,19 +972,19 @@ static inline void io_uring_prep_sync_file_range(struct io_uring_sqe *sqe, sqe->sync_range_flags = (__u32) flags; } -static inline void io_uring_prep_mkdirat(struct io_uring_sqe *sqe, int dfd, +IOURINGINLINE void io_uring_prep_mkdirat(struct io_uring_sqe *sqe, int dfd, const char *path, mode_t mode) { io_uring_prep_rw(IORING_OP_MKDIRAT, sqe, dfd, path, mode, 0); } -static inline void io_uring_prep_mkdir(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_mkdir(struct io_uring_sqe *sqe, const char *path, mode_t mode) { io_uring_prep_mkdirat(sqe, AT_FDCWD, path, mode); } -static inline void io_uring_prep_symlinkat(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_symlinkat(struct io_uring_sqe *sqe, const char *target, int newdirfd, const char *linkpath) { @@ -958,13 +992,14 @@ static inline void io_uring_prep_symlinkat(struct io_uring_sqe *sqe, (uint64_t) (uintptr_t) linkpath); } -static inline void io_uring_prep_symlink(struct io_uring_sqe *sqe, - const char *target, const char *linkpath) +IOURINGINLINE void io_uring_prep_symlink(struct io_uring_sqe *sqe, + const char *target, + const char *linkpath) { io_uring_prep_symlinkat(sqe, target, AT_FDCWD, linkpath); } -static inline void io_uring_prep_linkat(struct io_uring_sqe *sqe, int olddfd, +IOURINGINLINE void io_uring_prep_linkat(struct io_uring_sqe *sqe, int olddfd, const char *oldpath, int newdfd, const char *newpath, int flags) { @@ -973,25 +1008,55 @@ static inline void io_uring_prep_linkat(struct io_uring_sqe *sqe, int olddfd, sqe->hardlink_flags = (__u32) flags; } -static inline void io_uring_prep_link(struct io_uring_sqe *sqe, - const char *oldpath, const char *newpath, int flags) +IOURINGINLINE void io_uring_prep_link(struct io_uring_sqe *sqe, + const char *oldpath, const char *newpath, + int flags) { io_uring_prep_linkat(sqe, AT_FDCWD, oldpath, AT_FDCWD, newpath, flags); } -static inline void io_uring_prep_msg_ring(struct io_uring_sqe *sqe, int fd, +IOURINGINLINE void io_uring_prep_msg_ring_cqe_flags(struct io_uring_sqe *sqe, + int fd, unsigned int len, __u64 data, + unsigned int flags, unsigned int cqe_flags) +{ + io_uring_prep_rw(IORING_OP_MSG_RING, sqe, fd, NULL, len, data); + sqe->msg_ring_flags = IORING_MSG_RING_FLAGS_PASS | flags; + sqe->file_index = cqe_flags; +} + +IOURINGINLINE void io_uring_prep_msg_ring(struct io_uring_sqe *sqe, int fd, unsigned int len, __u64 data, unsigned int flags) { io_uring_prep_rw(IORING_OP_MSG_RING, sqe, fd, NULL, len, data); - sqe->rw_flags = flags; + sqe->msg_ring_flags = flags; } -static inline void io_uring_prep_getxattr(struct io_uring_sqe *sqe, - const char *name, - char *value, - const char *path, - unsigned int len) +IOURINGINLINE void io_uring_prep_msg_ring_fd(struct io_uring_sqe *sqe, int fd, + int source_fd, int target_fd, + __u64 data, unsigned int flags) +{ + io_uring_prep_rw(IORING_OP_MSG_RING, sqe, fd, + (void *) (uintptr_t) IORING_MSG_SEND_FD, 0, data); + sqe->addr3 = source_fd; + /* offset by 1 for allocation */ + if ((unsigned int) target_fd == IORING_FILE_INDEX_ALLOC) + target_fd--; + __io_uring_set_target_fixed_file(sqe, target_fd); + sqe->msg_ring_flags = flags; +} + +IOURINGINLINE void io_uring_prep_msg_ring_fd_alloc(struct io_uring_sqe *sqe, + int fd, int source_fd, + __u64 data, unsigned int flags) +{ + io_uring_prep_msg_ring_fd(sqe, fd, source_fd, IORING_FILE_INDEX_ALLOC, + data, flags); +} + +IOURINGINLINE void io_uring_prep_getxattr(struct io_uring_sqe *sqe, + const char *name, char *value, + const char *path, unsigned int len) { io_uring_prep_rw(IORING_OP_GETXATTR, sqe, 0, name, len, (__u64) (uintptr_t) value); @@ -999,11 +1064,9 @@ static inline void io_uring_prep_getxattr(struct io_uring_sqe *sqe, sqe->xattr_flags = 0; } -static inline void io_uring_prep_setxattr(struct io_uring_sqe *sqe, - const char *name, - const char *value, - const char *path, - int flags, +IOURINGINLINE void io_uring_prep_setxattr(struct io_uring_sqe *sqe, + const char *name, const char *value, + const char *path, int flags, unsigned int len) { io_uring_prep_rw(IORING_OP_SETXATTR, sqe, 0, name, len, @@ -1012,30 +1075,25 @@ static inline void io_uring_prep_setxattr(struct io_uring_sqe *sqe, sqe->xattr_flags = flags; } -static inline void io_uring_prep_fgetxattr(struct io_uring_sqe *sqe, - int fd, - const char *name, - char *value, - unsigned int len) +IOURINGINLINE void io_uring_prep_fgetxattr(struct io_uring_sqe *sqe, + int fd, const char *name, + char *value, unsigned int len) { io_uring_prep_rw(IORING_OP_FGETXATTR, sqe, fd, name, len, (__u64) (uintptr_t) value); sqe->xattr_flags = 0; } -static inline void io_uring_prep_fsetxattr(struct io_uring_sqe *sqe, - int fd, - const char *name, - const char *value, - int flags, - unsigned int len) +IOURINGINLINE void io_uring_prep_fsetxattr(struct io_uring_sqe *sqe, int fd, + const char *name, const char *value, + int flags, unsigned int len) { io_uring_prep_rw(IORING_OP_FSETXATTR, sqe, fd, name, len, (__u64) (uintptr_t) value); sqe->xattr_flags = flags; } -static inline void io_uring_prep_socket(struct io_uring_sqe *sqe, int domain, +IOURINGINLINE void io_uring_prep_socket(struct io_uring_sqe *sqe, int domain, int type, int protocol, unsigned int flags) { @@ -1043,7 +1101,7 @@ static inline void io_uring_prep_socket(struct io_uring_sqe *sqe, int domain, sqe->rw_flags = flags; } -static inline void io_uring_prep_socket_direct(struct io_uring_sqe *sqe, +IOURINGINLINE void io_uring_prep_socket_direct(struct io_uring_sqe *sqe, int domain, int type, int protocol, unsigned file_index, @@ -1051,12 +1109,16 @@ static inline void io_uring_prep_socket_direct(struct io_uring_sqe *sqe, { io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type); sqe->rw_flags = flags; + /* offset by 1 for allocation */ + if (file_index == IORING_FILE_INDEX_ALLOC) + file_index--; __io_uring_set_target_fixed_file(sqe, file_index); } -static inline void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe, - int domain, int type, int protocol, - unsigned int flags) +IOURINGINLINE void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe, + int domain, int type, + int protocol, + unsigned int flags) { io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type); sqe->rw_flags = flags; @@ -1067,7 +1129,7 @@ static inline void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe, * Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in * the SQ ring */ -static inline unsigned io_uring_sq_ready(const struct io_uring *ring) +IOURINGINLINE unsigned io_uring_sq_ready(const struct io_uring *ring) { unsigned khead = *ring->sq.khead; @@ -1086,7 +1148,7 @@ static inline unsigned io_uring_sq_ready(const struct io_uring *ring) /* * Returns how much space is left in the SQ ring. */ -static inline unsigned io_uring_sq_space_left(const struct io_uring *ring) +IOURINGINLINE unsigned io_uring_sq_space_left(const struct io_uring *ring) { return ring->sq.ring_entries - io_uring_sq_ready(ring); } @@ -1098,7 +1160,7 @@ static inline unsigned io_uring_sq_space_left(const struct io_uring *ring) * action is taken. Note: may return -EINVAL if the kernel doesn't support * this feature. */ -static inline int io_uring_sqring_wait(struct io_uring *ring) +IOURINGINLINE int io_uring_sqring_wait(struct io_uring *ring) { if (!(ring->flags & IORING_SETUP_SQPOLL)) return 0; @@ -1111,7 +1173,7 @@ static inline int io_uring_sqring_wait(struct io_uring *ring) /* * Returns how many unconsumed entries are ready in the CQ ring */ -static inline unsigned io_uring_cq_ready(const struct io_uring *ring) +IOURINGINLINE unsigned io_uring_cq_ready(const struct io_uring *ring) { return io_uring_smp_load_acquire(ring->cq.ktail) - *ring->cq.khead; } @@ -1120,7 +1182,7 @@ static inline unsigned io_uring_cq_ready(const struct io_uring *ring) * Returns true if there are overflow entries waiting to be flushed onto * the CQ ring */ -static inline bool io_uring_cq_has_overflow(const struct io_uring *ring) +IOURINGINLINE bool io_uring_cq_has_overflow(const struct io_uring *ring) { return IO_URING_READ_ONCE(*ring->sq.kflags) & IORING_SQ_CQ_OVERFLOW; } @@ -1128,7 +1190,7 @@ static inline bool io_uring_cq_has_overflow(const struct io_uring *ring) /* * Returns true if the eventfd notification is currently enabled */ -static inline bool io_uring_cq_eventfd_enabled(const struct io_uring *ring) +IOURINGINLINE bool io_uring_cq_eventfd_enabled(const struct io_uring *ring) { if (!ring->cq.kflags) return true; @@ -1140,7 +1202,7 @@ static inline bool io_uring_cq_eventfd_enabled(const struct io_uring *ring) * Toggle eventfd notification on or off, if an eventfd is registered with * the ring. */ -static inline int io_uring_cq_eventfd_toggle(struct io_uring *ring, +IOURINGINLINE int io_uring_cq_eventfd_toggle(struct io_uring *ring, bool enabled) { uint32_t flags; @@ -1168,7 +1230,7 @@ static inline int io_uring_cq_eventfd_toggle(struct io_uring *ring, * readily available. Returns 0 with cqe_ptr filled in on success, -errno on * failure. */ -static inline int io_uring_wait_cqe_nr(struct io_uring *ring, +IOURINGINLINE int io_uring_wait_cqe_nr(struct io_uring *ring, struct io_uring_cqe **cqe_ptr, unsigned wait_nr) { @@ -1180,7 +1242,7 @@ static inline int io_uring_wait_cqe_nr(struct io_uring *ring, * "official" versions of this, io_uring_peek_cqe(), io_uring_wait_cqe(), * or io_uring_wait_cqes*(). */ -static inline int __io_uring_peek_cqe(struct io_uring *ring, +IOURINGINLINE int __io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr, unsigned *nr_available) { @@ -1226,7 +1288,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring, * Return an IO completion, if one is readily available. Returns 0 with * cqe_ptr filled in on success, -errno on failure. */ -static inline int io_uring_peek_cqe(struct io_uring *ring, +IOURINGINLINE int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) { if (!__io_uring_peek_cqe(ring, cqe_ptr, NULL) && *cqe_ptr) @@ -1239,7 +1301,7 @@ static inline int io_uring_peek_cqe(struct io_uring *ring, * Return an IO completion, waiting for it if necessary. Returns 0 with * cqe_ptr filled in on success, -errno on failure. */ -static inline int io_uring_wait_cqe(struct io_uring *ring, +IOURINGINLINE int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) { if (!__io_uring_peek_cqe(ring, cqe_ptr, NULL) && *cqe_ptr) @@ -1255,7 +1317,7 @@ static inline int io_uring_wait_cqe(struct io_uring *ring, * * Returns a vacant sqe, or NULL if we're full. */ -static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring) +IOURINGINLINE struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring) { struct io_uring_sq *sq = &ring->sq; unsigned int head, next = sq->sqe_tail + 1; @@ -1282,12 +1344,12 @@ static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring) /* * Return the appropriate mask for a buffer ring of size 'ring_entries' */ -static inline int io_uring_buf_ring_mask(__u32 ring_entries) +IOURINGINLINE int io_uring_buf_ring_mask(__u32 ring_entries) { return ring_entries - 1; } -static inline void io_uring_buf_ring_init(struct io_uring_buf_ring *br) +IOURINGINLINE void io_uring_buf_ring_init(struct io_uring_buf_ring *br) { br->tail = 0; } @@ -1295,7 +1357,7 @@ static inline void io_uring_buf_ring_init(struct io_uring_buf_ring *br) /* * Assign 'buf' with the addr/len/buffer ID supplied */ -static inline void io_uring_buf_ring_add(struct io_uring_buf_ring *br, +IOURINGINLINE void io_uring_buf_ring_add(struct io_uring_buf_ring *br, void *addr, unsigned int len, unsigned short bid, int mask, int buf_offset) @@ -1312,7 +1374,7 @@ static inline void io_uring_buf_ring_add(struct io_uring_buf_ring *br, * io_uring_buf_ring_add() has been called 'count' times to fill in new * buffers. */ -static inline void io_uring_buf_ring_advance(struct io_uring_buf_ring *br, +IOURINGINLINE void io_uring_buf_ring_advance(struct io_uring_buf_ring *br, int count) { unsigned short new_tail = br->tail + count; @@ -1320,6 +1382,14 @@ static inline void io_uring_buf_ring_advance(struct io_uring_buf_ring *br, io_uring_smp_store_release(&br->tail, new_tail); } +IOURINGINLINE void __io_uring_buf_ring_cq_advance(struct io_uring *ring, + struct io_uring_buf_ring *br, + int cq_count, int buf_count) +{ + br->tail += buf_count; + io_uring_cq_advance(ring, cq_count); +} + /* * Make 'count' new buffers visible to the kernel while at the same time * advancing the CQ ring seen entries. This can be used when the application @@ -1327,16 +1397,15 @@ static inline void io_uring_buf_ring_advance(struct io_uring_buf_ring *br, * avoiding an extra atomic when needing to increment both the CQ ring and * the ring buffer index at the same time. */ -static inline void io_uring_buf_ring_cq_advance(struct io_uring *ring, +IOURINGINLINE void io_uring_buf_ring_cq_advance(struct io_uring *ring, struct io_uring_buf_ring *br, int count) { - br->tail += count; - io_uring_cq_advance(ring, count); + __io_uring_buf_ring_cq_advance(ring, br, count, count); } #ifndef LIBURING_INTERNAL -static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) +IOURINGINLINE struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) { return _io_uring_get_sqe(ring); } @@ -1347,8 +1416,30 @@ struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring); ssize_t io_uring_mlock_size(unsigned entries, unsigned flags); ssize_t io_uring_mlock_size_params(unsigned entries, struct io_uring_params *p); +/* + * Versioning information for liburing. + * + * Use IO_URING_CHECK_VERSION() for compile time checks including from + * preprocessor directives. + * + * Use io_uring_check_version() for runtime checks of the version of + * liburing that was loaded by the dynamic linker. + */ +int io_uring_major_version(void); +int io_uring_minor_version(void); +bool io_uring_check_version(int major, int minor); + +#define IO_URING_CHECK_VERSION(major,minor) \ + (major > IO_URING_VERSION_MAJOR || \ + (major == IO_URING_VERSION_MAJOR && \ + minor >= IO_URING_VERSION_MINOR)) + #ifdef __cplusplus } #endif +#ifdef IOURINGINLINE +#undef IOURINGINLINE +#endif + #endif diff --git a/contrib/libs/liburing/src/include/liburing/compat.h b/contrib/libs/liburing/src/include/liburing/compat.h index e579439578..717f81cb7f 100644 --- a/contrib/libs/liburing/src/include/liburing/compat.h +++ b/contrib/libs/liburing/src/include/liburing/compat.h @@ -3,6 +3,8 @@ #define LIBURING_COMPAT_H #include <linux/time_types.h> +/* <linux/time_types.h> is included above and not needed again */ +#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1 #include <linux/openat2.h> diff --git a/contrib/libs/liburing/src/include/liburing/io_uring.h b/contrib/libs/liburing/src/include/liburing/io_uring.h index a3e09208df..84c33b5e84 100644 --- a/contrib/libs/liburing/src/include/liburing/io_uring.h +++ b/contrib/libs/liburing/src/include/liburing/io_uring.h @@ -12,12 +12,11 @@ #include <linux/types.h> /* * this file is shared with liburing and that has to autodetect - * if linux/time_types.h is available + * if linux/time_types.h is available or not, it can + * define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H + * if linux/time_types.h is not available */ -#ifdef __KERNEL__ -#define HAVE_LINUX_TIME_TYPES_H 1 -#endif -#ifdef HAVE_LINUX_TIME_TYPES_H +#ifndef UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H #include <linux/time_types.h> #endif @@ -231,7 +230,7 @@ enum io_uring_op { /* * sqe->uring_cmd_flags - * IORING_URING_CMD_FIXED use registered buffer; pass thig flag + * IORING_URING_CMD_FIXED use registered buffer; pass this flag * along with setting sqe->buf_index. */ #define IORING_URING_CMD_FIXED (1U << 0) @@ -251,6 +250,7 @@ enum io_uring_op { #define IORING_TIMEOUT_REALTIME (1U << 3) #define IORING_LINK_TIMEOUT_UPDATE (1U << 4) #define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5) +#define IORING_TIMEOUT_MULTISHOT (1U << 6) #define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) #define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) /* @@ -305,10 +305,28 @@ enum io_uring_op { * * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in * the buf_index field. + * + * IORING_SEND_ZC_REPORT_USAGE + * If set, SEND[MSG]_ZC should report + * the zerocopy usage in cqe.res + * for the IORING_CQE_F_NOTIF cqe. + * 0 is reported if zerocopy was actually possible. + * IORING_NOTIF_USAGE_ZC_COPIED if data was copied + * (at least partially). */ #define IORING_RECVSEND_POLL_FIRST (1U << 0) #define IORING_RECV_MULTISHOT (1U << 1) #define IORING_RECVSEND_FIXED_BUF (1U << 2) +#define IORING_SEND_ZC_REPORT_USAGE (1U << 3) + +/* + * cqe.res for IORING_CQE_F_NOTIF if + * IORING_SEND_ZC_REPORT_USAGE was requested + * + * It should be treated as a flag, all other + * bits of cqe.res should be treated as reserved! + */ +#define IORING_NOTIF_USAGE_ZC_COPIED (1U << 31) /* * accept flags stored in sqe->ioprio @@ -330,6 +348,8 @@ enum { * applicable for IORING_MSG_DATA, obviously. */ #define IORING_MSG_RING_CQE_SKIP (1U << 0) +/* Pass through the flags from sqe->file_index to cqe->flags */ +#define IORING_MSG_RING_FLAGS_PASS (1U << 1) /* * IO completion data structure (Completion Queue Entry) @@ -370,6 +390,9 @@ enum { #define IORING_OFF_SQ_RING 0ULL #define IORING_OFF_CQ_RING 0x8000000ULL #define IORING_OFF_SQES 0x10000000ULL +#define IORING_OFF_PBUF_RING 0x80000000ULL +#define IORING_OFF_PBUF_SHIFT 16 +#define IORING_OFF_MMAP_MASK 0xf8000000ULL /* * Filled with the offset for mmap(2) @@ -453,6 +476,7 @@ struct io_uring_params { #define IORING_FEAT_RSRC_TAGS (1U << 10) #define IORING_FEAT_CQE_SKIP (1U << 11) #define IORING_FEAT_LINKED_FILE (1U << 12) +#define IORING_FEAT_REG_REG_RING (1U << 13) /* * io_uring_register(2) opcodes and arguments @@ -500,7 +524,10 @@ enum { IORING_REGISTER_FILE_ALLOC_RANGE = 25, /* this goes last */ - IORING_REGISTER_LAST + IORING_REGISTER_LAST, + + /* flag added to the opcode to use a registered ring fd */ + IORING_REGISTER_USE_REGISTERED_RING = 1U << 31 }; /* io-wq worker categories */ @@ -545,19 +572,6 @@ struct io_uring_rsrc_update2 { __u32 resv2; }; -struct io_uring_notification_slot { - __u64 tag; - __u64 resv[3]; -}; - -struct io_uring_notification_register { - __u32 nr_slots; - __u32 resv; - __u64 resv2; - __u64 data; - __u64 resv3; -}; - /* Skip updating fd indexes set to this value in the fd table */ #define IORING_REGISTER_FILES_SKIP (-2) @@ -612,12 +626,26 @@ struct io_uring_buf_ring { }; }; +/* + * Flags for IORING_REGISTER_PBUF_RING. + * + * IOU_PBUF_RING_MMAP: If set, kernel will allocate the memory for the ring. + * The application must not set a ring_addr in struct + * io_uring_buf_reg, instead it must subsequently call + * mmap(2) with the offset set as: + * IORING_OFF_PBUF_RING | (bgid << IORING_OFF_PBUF_SHIFT) + * to get a virtual mapping for the ring. + */ +enum { + IOU_PBUF_RING_MMAP = 1, +}; + /* argument for IORING_(UN)REGISTER_PBUF_RING */ struct io_uring_buf_reg { __u64 ring_addr; __u32 ring_entries; __u16 bgid; - __u16 pad; + __u16 flags; __u64 resv[3]; }; diff --git a/contrib/libs/liburing/src/include/liburing/io_uring_version.h b/contrib/libs/liburing/src/include/liburing/io_uring_version.h new file mode 100644 index 0000000000..8029e041f9 --- /dev/null +++ b/contrib/libs/liburing/src/include/liburing/io_uring_version.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef LIBURING_VERSION_H +#define LIBURING_VERSION_H + +#define IO_URING_VERSION_MAJOR 2 +#define IO_URING_VERSION_MINOR 4 + +#endif diff --git a/contrib/libs/liburing/src/int_flags.h b/contrib/libs/liburing/src/int_flags.h index 90505ec340..71774fbca5 100644 --- a/contrib/libs/liburing/src/int_flags.h +++ b/contrib/libs/liburing/src/int_flags.h @@ -4,6 +4,7 @@ enum { INT_FLAG_REG_RING = 1, + INT_FLAG_REG_REG_RING = 2, }; #endif diff --git a/contrib/libs/liburing/src/lib.h b/contrib/libs/liburing/src/lib.h index 82c0ab10d3..635a30ece5 100644 --- a/contrib/libs/liburing/src/lib.h +++ b/contrib/libs/liburing/src/lib.h @@ -37,25 +37,14 @@ #define __hot __attribute__((__hot__)) #define __cold __attribute__((__cold__)) +#ifdef CONFIG_NOLIBC +void *__uring_memset(void *s, int c, size_t n); void *__uring_malloc(size_t len); void __uring_free(void *p); -static inline void *uring_malloc(size_t len) -{ -#ifdef CONFIG_NOLIBC - return __uring_malloc(len); -#else - return malloc(len); -#endif -} - -static inline void uring_free(void *ptr) -{ -#ifdef CONFIG_NOLIBC - __uring_free(ptr); -#else - free(ptr); +#define malloc(LEN) __uring_malloc(LEN) +#define free(PTR) __uring_free(PTR) +#define memset(PTR, C, LEN) __uring_memset(PTR, C, LEN) #endif -} #endif /* #ifndef LIBURING_LIB_H */ diff --git a/contrib/libs/liburing/src/nolibc.c b/contrib/libs/liburing/src/nolibc.c new file mode 100644 index 0000000000..22d23ec50d --- /dev/null +++ b/contrib/libs/liburing/src/nolibc.c @@ -0,0 +1,56 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +#ifndef CONFIG_NOLIBC +#error "This file should only be compiled for no libc build" +#endif + +#include "lib.h" +#include "syscall.h" + +void *__uring_memset(void *s, int c, size_t n) +{ + size_t i; + unsigned char *p = s; + + for (i = 0; i < n; i++) { + p[i] = (unsigned char) c; + + /* + * An empty inline ASM to avoid auto-vectorization + * because it's too bloated for liburing. + */ + __asm__ volatile (""); + } + + return s; +} + +struct uring_heap { + size_t len; + char user_p[] __attribute__((__aligned__)); +}; + +void *__uring_malloc(size_t len) +{ + struct uring_heap *heap; + + heap = __sys_mmap(NULL, sizeof(*heap) + len, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (IS_ERR(heap)) + return NULL; + + heap->len = sizeof(*heap) + len; + return heap->user_p; +} + +void __uring_free(void *p) +{ + struct uring_heap *heap; + + if (uring_unlikely(!p)) + return; + + heap = container_of(p, struct uring_heap, user_p); + __sys_munmap(heap, heap->len); +} diff --git a/contrib/libs/liburing/src/queue.c b/contrib/libs/liburing/src/queue.c index 4b4220782d..9fca31fe34 100644 --- a/contrib/libs/liburing/src/queue.c +++ b/contrib/libs/liburing/src/queue.c @@ -82,7 +82,7 @@ static int _io_uring_get_cqe(struct io_uring *ring, } if (!cqe && !data->wait_nr && !data->submit) { /* - * If we already looped once, we already entererd + * If we already looped once, we already entered * the kernel. Since there's nothing to submit or * wait for, don't keep retrying. */ @@ -202,7 +202,7 @@ again: * Sync internal state with kernel ring state on the SQ side. Returns the * number of pending items in the SQ ring, for the shared ring. */ -unsigned __io_uring_flush_sq(struct io_uring *ring) +static unsigned __io_uring_flush_sq(struct io_uring *ring) { struct io_uring_sq *sq = &ring->sq; unsigned tail = sq->sqe_tail; diff --git a/contrib/libs/liburing/src/register.c b/contrib/libs/liburing/src/register.c index 13e80a5811..5563db2c4a 100644 --- a/contrib/libs/liburing/src/register.c +++ b/contrib/libs/liburing/src/register.c @@ -9,6 +9,15 @@ #include "liburing/compat.h" #include "liburing/io_uring.h" +static inline int do_register(struct io_uring *ring, unsigned int opcode, + const void *arg, unsigned int nr_args) +{ + if (ring->int_flags & INT_FLAG_REG_REG_RING) + opcode |= IORING_REGISTER_USE_REGISTERED_RING; + + return __sys_io_uring_register(ring->enter_ring_fd, opcode, arg, nr_args); +} + int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off, const struct iovec *iovecs, const __u64 *tags, @@ -21,8 +30,7 @@ int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off, .nr = nr, }; - return __sys_io_uring_register(ring->ring_fd,IORING_REGISTER_BUFFERS_UPDATE, &up, - sizeof(up)); + return do_register(ring, IORING_REGISTER_BUFFERS_UPDATE, &up, sizeof(up)); } int io_uring_register_buffers_tags(struct io_uring *ring, @@ -36,9 +44,7 @@ int io_uring_register_buffers_tags(struct io_uring *ring, .tags = (unsigned long)tags, }; - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_BUFFERS2, ®, - sizeof(reg)); + return do_register(ring, IORING_REGISTER_BUFFERS2, ®, sizeof(reg)); } int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr) @@ -48,27 +54,18 @@ int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr) .nr = nr, }; - return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS2, - ®, sizeof(reg)); + return do_register(ring, IORING_REGISTER_BUFFERS2, ®, sizeof(reg)); } int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs, unsigned nr_iovecs) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS, - iovecs, nr_iovecs); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_REGISTER_BUFFERS, iovecs, nr_iovecs); } int io_uring_unregister_buffers(struct io_uring *ring) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS, - NULL, 0); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_UNREGISTER_BUFFERS, NULL, 0); } int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off, @@ -82,9 +79,7 @@ int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off, .nr = nr_files, }; - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_FILES_UPDATE2, &up, - sizeof(up)); + return do_register(ring, IORING_REGISTER_FILES_UPDATE2, &up, sizeof(up)); } /* @@ -102,9 +97,7 @@ int io_uring_register_files_update(struct io_uring *ring, unsigned off, .fds = (unsigned long) files, }; - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_FILES_UPDATE, &up, - nr_files); + return do_register(ring, IORING_REGISTER_FILES_UPDATE, &up, nr_files); } static int increase_rlimit_nofile(unsigned nr) @@ -133,9 +126,7 @@ int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr) int ret, did_increase = 0; do { - ret = __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_FILES2, ®, - sizeof(reg)); + ret = do_register(ring, IORING_REGISTER_FILES2, ®, sizeof(reg)); if (ret >= 0) break; if (ret == -EMFILE && !did_increase) { @@ -160,9 +151,7 @@ int io_uring_register_files_tags(struct io_uring *ring, const int *files, int ret, did_increase = 0; do { - ret = __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_FILES2, ®, - sizeof(reg)); + ret = do_register(ring, IORING_REGISTER_FILES2, ®, sizeof(reg)); if (ret >= 0) break; if (ret == -EMFILE && !did_increase) { @@ -182,9 +171,7 @@ int io_uring_register_files(struct io_uring *ring, const int *files, int ret, did_increase = 0; do { - ret = __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_FILES, files, - nr_files); + ret = do_register(ring, IORING_REGISTER_FILES, files, nr_files); if (ret >= 0) break; if (ret == -EMFILE && !did_increase) { @@ -200,79 +187,50 @@ int io_uring_register_files(struct io_uring *ring, const int *files, int io_uring_unregister_files(struct io_uring *ring) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES, - NULL, 0); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_UNREGISTER_FILES, NULL, 0); } int io_uring_register_eventfd(struct io_uring *ring, int event_fd) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD, - &event_fd, 1); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_REGISTER_EVENTFD, &event_fd, 1); } int io_uring_unregister_eventfd(struct io_uring *ring) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD, - NULL, 0); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_UNREGISTER_EVENTFD, NULL, 0); } int io_uring_register_eventfd_async(struct io_uring *ring, int event_fd) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_EVENTFD_ASYNC, &event_fd, - 1); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_REGISTER_EVENTFD_ASYNC, &event_fd, 1); } int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p, unsigned int nr_ops) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE, p, - nr_ops); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_REGISTER_PROBE, p, nr_ops); } int io_uring_register_personality(struct io_uring *ring) { - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_PERSONALITY, NULL, 0); + return do_register(ring, IORING_REGISTER_PERSONALITY, NULL, 0); } int io_uring_unregister_personality(struct io_uring *ring, int id) { - return __sys_io_uring_register(ring->ring_fd, - IORING_UNREGISTER_PERSONALITY, NULL, id); + return do_register(ring, IORING_UNREGISTER_PERSONALITY, NULL, id); } int io_uring_register_restrictions(struct io_uring *ring, struct io_uring_restriction *res, unsigned int nr_res) { - int ret; - - ret = __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_RESTRICTIONS, res, - nr_res); - return (ret < 0) ? ret : 0; + return do_register(ring, IORING_REGISTER_RESTRICTIONS, res, nr_res); } int io_uring_enable_rings(struct io_uring *ring) { - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_ENABLE_RINGS, NULL, 0); + return do_register(ring, IORING_REGISTER_ENABLE_RINGS, NULL, 0); } int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz, @@ -281,21 +239,17 @@ int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz, if (cpusz >= (1U << 31)) return -EINVAL; - return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_IOWQ_AFF, - mask, (int) cpusz); + return do_register(ring, IORING_REGISTER_IOWQ_AFF, mask, (int) cpusz); } int io_uring_unregister_iowq_aff(struct io_uring *ring) { - return __sys_io_uring_register(ring->ring_fd, - IORING_UNREGISTER_IOWQ_AFF, NULL, 0); + return do_register(ring, IORING_UNREGISTER_IOWQ_AFF, NULL, 0); } int io_uring_register_iowq_max_workers(struct io_uring *ring, unsigned int *val) { - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_IOWQ_MAX_WORKERS, val, - 2); + return do_register(ring, IORING_REGISTER_IOWQ_MAX_WORKERS, val, 2); } int io_uring_register_ring_fd(struct io_uring *ring) @@ -306,11 +260,16 @@ int io_uring_register_ring_fd(struct io_uring *ring) }; int ret; - ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RING_FDS, - &up, 1); + if (ring->int_flags & INT_FLAG_REG_RING) + return -EEXIST; + + ret = do_register(ring, IORING_REGISTER_RING_FDS, &up, 1); if (ret == 1) { ring->enter_ring_fd = up.offset; ring->int_flags |= INT_FLAG_REG_RING; + if (ring->features & IORING_FEAT_REG_REG_RING) { + ring->int_flags |= INT_FLAG_REG_REG_RING; + } } return ret; } @@ -323,48 +282,58 @@ int io_uring_unregister_ring_fd(struct io_uring *ring) }; int ret; - ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_RING_FDS, - &up, 1); + if (!(ring->int_flags & INT_FLAG_REG_RING)) + return -EINVAL; + + ret = do_register(ring, IORING_UNREGISTER_RING_FDS, &up, 1); if (ret == 1) { ring->enter_ring_fd = ring->ring_fd; - ring->int_flags &= ~INT_FLAG_REG_RING; + ring->int_flags &= ~(INT_FLAG_REG_RING | INT_FLAG_REG_REG_RING); } return ret; } +int io_uring_close_ring_fd(struct io_uring *ring) +{ + if (!(ring->features & IORING_FEAT_REG_REG_RING)) + return -EOPNOTSUPP; + if (!(ring->int_flags & INT_FLAG_REG_RING)) + return -EINVAL; + if (ring->ring_fd == -1) + return -EBADF; + + __sys_close(ring->ring_fd); + ring->ring_fd = -1; + return 1; +} + int io_uring_register_buf_ring(struct io_uring *ring, struct io_uring_buf_reg *reg, unsigned int __maybe_unused flags) { - return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PBUF_RING, - reg, 1); + return do_register(ring, IORING_REGISTER_PBUF_RING, reg, 1); } int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid) { struct io_uring_buf_reg reg = { .bgid = bgid }; - return __sys_io_uring_register(ring->ring_fd, - IORING_UNREGISTER_PBUF_RING, ®, 1); + return do_register(ring, IORING_UNREGISTER_PBUF_RING, ®, 1); } int io_uring_register_sync_cancel(struct io_uring *ring, struct io_uring_sync_cancel_reg *reg) { - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_SYNC_CANCEL, reg, 1); + return do_register(ring, IORING_REGISTER_SYNC_CANCEL, reg, 1); } int io_uring_register_file_alloc_range(struct io_uring *ring, unsigned off, unsigned len) { - struct io_uring_file_index_range range; - - memset(&range, 0, sizeof(range)); - range.off = off; - range.len = len; + struct io_uring_file_index_range range = { + .off = off, + .len = len + }; - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_FILE_ALLOC_RANGE, &range, - 0); + return do_register(ring, IORING_REGISTER_FILE_ALLOC_RANGE, &range, 0); } diff --git a/contrib/libs/liburing/src/setup.c b/contrib/libs/liburing/src/setup.c index 0c15e75c1c..db2f3dfe15 100644 --- a/contrib/libs/liburing/src/setup.c +++ b/contrib/libs/liburing/src/setup.c @@ -106,6 +106,7 @@ __cold int io_uring_queue_mmap(int fd, struct io_uring_params *p, ring->flags = p->flags; ring->ring_fd = ring->enter_ring_fd = fd; ring->int_flags = 0; + return 0; } return ret; } @@ -206,7 +207,8 @@ __cold void io_uring_queue_exit(struct io_uring *ring) */ if (ring->int_flags & INT_FLAG_REG_RING) io_uring_unregister_ring_fd(ring); - __sys_close(ring->ring_fd); + if (ring->ring_fd != -1) + __sys_close(ring->ring_fd); } __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring) @@ -216,7 +218,7 @@ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring) int r; len = sizeof(*probe) + 256 * sizeof(struct io_uring_probe_op); - probe = uring_malloc(len); + probe = malloc(len); if (!probe) return NULL; memset(probe, 0, len); @@ -225,7 +227,7 @@ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring) if (r >= 0) return probe; - uring_free(probe); + free(probe); return NULL; } @@ -246,7 +248,7 @@ __cold struct io_uring_probe *io_uring_get_probe(void) __cold void io_uring_free_probe(struct io_uring_probe *probe) { - uring_free(probe); + free(probe); } static inline int __fls(unsigned long x) @@ -305,12 +307,14 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries, __cold ssize_t io_uring_mlock_size_params(unsigned entries, struct io_uring_params *p) { - struct io_uring_params lp = { }; + struct io_uring_params lp; struct io_uring ring; unsigned cq_entries; long page_size; ssize_t ret; + memset(&lp, 0, sizeof(lp)); + /* * We only really use this inited ring to see if the kernel is newer * or not. Newer kernels don't require memlocked memory. If we fail, @@ -364,7 +368,105 @@ __cold ssize_t io_uring_mlock_size_params(unsigned entries, */ __cold ssize_t io_uring_mlock_size(unsigned entries, unsigned flags) { - struct io_uring_params p = { .flags = flags, }; + struct io_uring_params p; + memset(&p, 0, sizeof(p)); + p.flags = flags; return io_uring_mlock_size_params(entries, &p); } + +#if defined(__hppa__) +static struct io_uring_buf_ring *br_setup(struct io_uring *ring, + unsigned int nentries, int bgid, + unsigned int flags, int *ret) +{ + struct io_uring_buf_ring *br; + struct io_uring_buf_reg reg; + size_t ring_size; + off_t off; + int lret; + + memset(®, 0, sizeof(reg)); + reg.ring_entries = nentries; + reg.bgid = bgid; + reg.flags = IOU_PBUF_RING_MMAP; + + *ret = 0; + lret = io_uring_register_buf_ring(ring, ®, flags); + if (lret) { + *ret = lret; + return NULL; + } + + off = IORING_OFF_PBUF_RING | (unsigned long long) bgid << IORING_OFF_PBUF_SHIFT; + ring_size = nentries * sizeof(struct io_uring_buf); + br = __sys_mmap(NULL, ring_size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, ring->ring_fd, off); + if (IS_ERR(br)) { + *ret = PTR_ERR(br); + return NULL; + } + + return br; +} +#else +static struct io_uring_buf_ring *br_setup(struct io_uring *ring, + unsigned int nentries, int bgid, + unsigned int flags, int *ret) +{ + struct io_uring_buf_ring *br; + struct io_uring_buf_reg reg; + size_t ring_size; + int lret; + + memset(®, 0, sizeof(reg)); + ring_size = nentries * sizeof(struct io_uring_buf); + br = __sys_mmap(NULL, ring_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (IS_ERR(br)) { + *ret = PTR_ERR(br); + return NULL; + } + + reg.ring_addr = (unsigned long) (uintptr_t) br; + reg.ring_entries = nentries; + reg.bgid = bgid; + + *ret = 0; + lret = io_uring_register_buf_ring(ring, ®, flags); + if (lret) { + __sys_munmap(br, ring_size); + *ret = lret; + br = NULL; + } + + return br; +} +#endif + +struct io_uring_buf_ring *io_uring_setup_buf_ring(struct io_uring *ring, + unsigned int nentries, + int bgid, unsigned int flags, + int *ret) +{ + struct io_uring_buf_ring *br; + + br = br_setup(ring, nentries, bgid, flags, ret); + if (br) + io_uring_buf_ring_init(br); + + return br; +} + +int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br, + unsigned int nentries, int bgid) +{ + int ret; + + ret = io_uring_unregister_buf_ring(ring, bgid); + if (ret) + return ret; + + __sys_munmap(br, nentries * sizeof(struct io_uring_buf)); + return 0; +} diff --git a/contrib/libs/liburing/src/version.c b/contrib/libs/liburing/src/version.c new file mode 100644 index 0000000000..e1a01229be --- /dev/null +++ b/contrib/libs/liburing/src/version.c @@ -0,0 +1,22 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ + +#include "liburing.h" +#include "liburing/io_uring_version.h" + +int io_uring_major_version(void) +{ + return IO_URING_VERSION_MAJOR; +} + +int io_uring_minor_version(void) +{ + return IO_URING_VERSION_MINOR; +} + +bool io_uring_check_version(int major, int minor) +{ + return major > io_uring_major_version() || + (major == io_uring_major_version() && + minor >= io_uring_minor_version()); +} diff --git a/contrib/libs/liburing/test/232c93d07b74.c b/contrib/libs/liburing/test/232c93d07b74.c index ab28adab17..a89e7aebbf 100644 --- a/contrib/libs/liburing/test/232c93d07b74.c +++ b/contrib/libs/liburing/test/232c93d07b74.c @@ -34,9 +34,9 @@ struct params { __be16 bind_port; }; -pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t cond = PTHREAD_COND_INITIALIZER; -int rcv_ready = 0; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +static int rcv_ready = 0; static void set_rcv_ready(void) { diff --git a/contrib/libs/liburing/test/35fa71a030ca.c b/contrib/libs/liburing/test/35fa71a030ca.c index cf8e3ff605..b21f95a54e 100644 --- a/contrib/libs/liburing/test/35fa71a030ca.c +++ b/contrib/libs/liburing/test/35fa71a030ca.c @@ -177,7 +177,7 @@ static void kill_and_wait(int pid, int* status) } #define SYZ_HAVE_SETUP_TEST 1 -static void setup_test() +static void setup_test(void) { prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); setpgrp(); @@ -263,7 +263,7 @@ static void loop(void) } } -uint64_t r[1] = {0xffffffffffffffff}; +static uint64_t r[1] = {0xffffffffffffffff}; void execute_call(int call) { @@ -321,7 +321,7 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; signal(SIGINT, sig_int); - mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0); signal(SIGALRM, sig_int); alarm(5); diff --git a/contrib/libs/liburing/test/500f9fbadef8.c b/contrib/libs/liburing/test/500f9fbadef8.c index c9e4397820..5306d09431 100644 --- a/contrib/libs/liburing/test/500f9fbadef8.c +++ b/contrib/libs/liburing/test/500f9fbadef8.c @@ -43,6 +43,8 @@ int main(int argc, char *argv[]) sprintf(buf, "./XXXXXX"); fd = mkostemp(buf, O_WRONLY | O_DIRECT | O_CREAT); if (fd < 0) { + if (errno == EINVAL) + return T_EXIT_SKIP; perror("mkostemp"); return T_EXIT_FAIL; } diff --git a/contrib/libs/liburing/test/917257daa0fe.c b/contrib/libs/liburing/test/917257daa0fe.c index b24d140d83..0fc8e57178 100644 --- a/contrib/libs/liburing/test/917257daa0fe.c +++ b/contrib/libs/liburing/test/917257daa0fe.c @@ -20,7 +20,7 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; - mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0); *(uint32_t*)0x20000000 = 0; *(uint32_t*)0x20000004 = 0; diff --git a/contrib/libs/liburing/test/a0908ae19763.c b/contrib/libs/liburing/test/a0908ae19763.c index dbe9c5100a..0b0f6a0eac 100644 --- a/contrib/libs/liburing/test/a0908ae19763.c +++ b/contrib/libs/liburing/test/a0908ae19763.c @@ -15,13 +15,13 @@ #include "helpers.h" #include "../src/syscall.h" -uint64_t r[1] = {0xffffffffffffffff}; +static uint64_t r[1] = {0xffffffffffffffff}; int main(int argc, char *argv[]) { if (argc > 1) return T_EXIT_SKIP; - mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0); intptr_t res = 0; *(uint32_t*)0x20000080 = 0; *(uint32_t*)0x20000084 = 0; diff --git a/contrib/libs/liburing/test/a4c0b3decb33.c b/contrib/libs/liburing/test/a4c0b3decb33.c index 376ecb3d87..89a750a471 100644 --- a/contrib/libs/liburing/test/a4c0b3decb33.c +++ b/contrib/libs/liburing/test/a4c0b3decb33.c @@ -96,7 +96,7 @@ static void kill_and_wait(int pid, int* status) } } -static void setup_test() +static void setup_test(void) { prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); setpgrp(); @@ -176,7 +176,7 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; signal(SIGINT, sig_int); - mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0); loop(); return T_EXIT_PASS; } diff --git a/contrib/libs/liburing/test/accept-link.c b/contrib/libs/liburing/test/accept-link.c index af47e0f4f0..36b8e3dd6e 100644 --- a/contrib/libs/liburing/test/accept-link.c +++ b/contrib/libs/liburing/test/accept-link.c @@ -17,8 +17,8 @@ #include "liburing.h" #include "helpers.h" -pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; static int recv_thread_ready = 0; static int recv_thread_done = 0; @@ -78,7 +78,7 @@ static void *send_thread(void *arg) return NULL; } -void *recv_thread(void *arg) +static void *recv_thread(void *arg) { struct data *data = arg; struct io_uring ring; @@ -196,7 +196,7 @@ static int test_accept_timeout(int do_connect, unsigned long timeout) if (ret) { fprintf(stderr, "queue_init: %d\n", ret); return 1; - }; + } fast_poll = (p.features & IORING_FEAT_FAST_POLL) != 0; io_uring_queue_exit(&ring); diff --git a/contrib/libs/liburing/test/accept-reuse.c b/contrib/libs/liburing/test/accept-reuse.c index fb15ade840..47c1b1361f 100644 --- a/contrib/libs/liburing/test/accept-reuse.c +++ b/contrib/libs/liburing/test/accept-reuse.c @@ -1,6 +1,5 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ -#include <liburing.h> #include <netdb.h> #include <string.h> #include <sys/socket.h> @@ -12,17 +11,16 @@ #include "helpers.h" #include "../src/syscall.h" -struct io_uring io_uring; +static struct io_uring io_uring; -int sys_io_uring_enter(const int fd, - const unsigned to_submit, - const unsigned min_complete, - const unsigned flags, sigset_t * const sig) +static int sys_io_uring_enter(const int fd, const unsigned to_submit, + const unsigned min_complete, + const unsigned flags, sigset_t * const sig) { return __sys_io_uring_enter(fd, to_submit, min_complete, flags, sig); } -int submit_sqe(void) +static int submit_sqe(void) { struct io_uring_sq *sq = &io_uring.sq; const unsigned tail = *sq->ktail; diff --git a/contrib/libs/liburing/test/accept.c b/contrib/libs/liburing/test/accept.c index cde1542a20..df23eab53f 100644 --- a/contrib/libs/liburing/test/accept.c +++ b/contrib/libs/liburing/test/accept.c @@ -273,6 +273,7 @@ static int test_loop(struct io_uring *ring, uint32_t multishot_mask = 0; int nr_fds = multishot ? MAX_FDS : 1; int multishot_idx = multishot ? INITIAL_USER_DATA : 0; + int err_ret = T_EXIT_FAIL; if (args.overflow) cause_overflow(ring); @@ -299,6 +300,7 @@ static int test_loop(struct io_uring *ring, no_accept_multi = 1; else no_accept = 1; + ret = T_EXIT_SKIP; goto out; } else if (s_fd[i] < 0) { if (args.accept_should_error && @@ -361,10 +363,10 @@ static int test_loop(struct io_uring *ring, out: close_sock_fds(s_fd, c_fd, nr_fds, fixed); - return 0; + return T_EXIT_PASS; err: close_sock_fds(s_fd, c_fd, nr_fds, fixed); - return 1; + return err_ret; } static int test(struct io_uring *ring, struct accept_test_args args) @@ -373,7 +375,7 @@ static int test(struct io_uring *ring, struct accept_test_args args) int ret = 0; int loop; int32_t recv_s0 = start_accept_listen(&addr, 0, - args.nonblock ? O_NONBLOCK : 0); + args.nonblock ? SOCK_NONBLOCK : 0); if (args.queue_accept_before_connect) queue_accept_conn(ring, recv_s0, args); for (loop = 0; loop < 1 + args.extra_loops; loop++) { @@ -459,7 +461,7 @@ static int test_accept_many(struct test_accept_many_args args) for (i = 0; i < nr_socks; i++) fds[i] = start_accept_listen(NULL, i, - args.nonblock ? O_NONBLOCK : 0); + args.nonblock ? SOCK_NONBLOCK : 0); for (i = 0; i < nr; i++) { int sock_idx = args.single_sock ? 0 : i; @@ -612,7 +614,7 @@ static int test_multishot_accept(int count, bool before, bool overflow) return ret; } -static int test_accept_multishot_wrong_arg() +static int test_accept_multishot_wrong_arg(void) { struct io_uring m_io_uring; struct io_uring_cqe *cqe; @@ -734,8 +736,9 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; + ret = test_accept(1, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept failed\n"); return ret; } @@ -743,141 +746,141 @@ int main(int argc, char *argv[]) return T_EXIT_SKIP; ret = test_accept(2, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept(2) failed\n"); return ret; } ret = test_accept(2, true); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept(2, true) failed\n"); return ret; } ret = test_accept_nonblock(false, 1); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_nonblock failed\n"); return ret; } ret = test_accept_nonblock(true, 1); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_nonblock(before, 1) failed\n"); return ret; } ret = test_accept_nonblock(true, 3); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_nonblock(before,3) failed\n"); return ret; } ret = test_accept_fixed(); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_fixed failed\n"); return ret; } ret = test_multishot_fixed_accept(); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_multishot_fixed_accept failed\n"); return ret; } ret = test_accept_multishot_wrong_arg(); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_multishot_wrong_arg failed\n"); return ret; } ret = test_accept_sqpoll(); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_sqpoll failed\n"); return ret; } ret = test_accept_cancel(0, 1, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel nodelay failed\n"); return ret; } ret = test_accept_cancel(10000, 1, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel delay failed\n"); return ret; } ret = test_accept_cancel(0, 4, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel nodelay failed\n"); return ret; } ret = test_accept_cancel(10000, 4, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel delay failed\n"); return ret; } ret = test_accept_cancel(0, 1, true); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel multishot nodelay failed\n"); return ret; } ret = test_accept_cancel(10000, 1, true); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel multishot delay failed\n"); return ret; } ret = test_accept_cancel(0, 4, true); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel multishot nodelay failed\n"); return ret; } ret = test_accept_cancel(10000, 4, true); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_cancel multishot delay failed\n"); return ret; } ret = test_multishot_accept(1, true, true); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_multishot_accept(1, false, true) failed\n"); return ret; } ret = test_multishot_accept(1, false, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_multishot_accept(1, false, false) failed\n"); return ret; } ret = test_multishot_accept(1, true, false); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_multishot_accept(1, true, false) failed\n"); return ret; } ret = test_accept_many((struct test_accept_many_args) {}); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_many failed\n"); return ret; } ret = test_accept_many((struct test_accept_many_args) { .usecs = 100000 }); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_many(sleep) failed\n"); return ret; } ret = test_accept_many((struct test_accept_many_args) { .nonblock = true }); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_many(nonblock) failed\n"); return ret; } @@ -886,13 +889,13 @@ int main(int argc, char *argv[]) .nonblock = true, .single_sock = true, .close_fds = true }); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_many(nonblock,close) failed\n"); return ret; } ret = test_accept_pending_on_exit(); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_accept_pending_on_exit failed\n"); return ret; } diff --git a/contrib/libs/liburing/test/b19062a56726.c b/contrib/libs/liburing/test/b19062a56726.c index e065e30f6e..9356769818 100644 --- a/contrib/libs/liburing/test/b19062a56726.c +++ b/contrib/libs/liburing/test/b19062a56726.c @@ -20,7 +20,7 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; - mmap((void *) 0x20000000, 0x1000000, 3, 0x32, -1, 0); + mmap((void *) 0x20000000, 0x1000000, 3, MAP_ANON|MAP_PRIVATE, -1, 0); *(uint32_t*)0x20000200 = 0; *(uint32_t*)0x20000204 = 0; diff --git a/contrib/libs/liburing/test/buf-ring.c b/contrib/libs/liburing/test/buf-ring.c index 569639a347..08f03f99ec 100644 --- a/contrib/libs/liburing/test/buf-ring.c +++ b/contrib/libs/liburing/test/buf-ring.c @@ -10,20 +10,22 @@ #include <stdlib.h> #include <string.h> #include <fcntl.h> +#include <sys/mman.h> #include "liburing.h" #include "helpers.h" static int no_buf_ring; +static int pagesize; /* test trying to register classic group when ring group exists */ static int test_mixed_reg2(int bgid) { - struct io_uring_buf_reg reg = { }; + struct io_uring_buf_ring *br; struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; struct io_uring ring; - void *ptr, *bufs; + void *bufs; int ret; ret = t_create_ring(1, &ring, 0); @@ -32,15 +34,8 @@ static int test_mixed_reg2(int bgid) else if (ret != T_SETUP_OK) return 1; - if (posix_memalign(&ptr, 4096, 4096)) - return 1; - - reg.ring_addr = (unsigned long) ptr; - reg.ring_entries = 32; - reg.bgid = bgid; - - ret = io_uring_register_buf_ring(&ring, ®, 0); - if (ret) { + br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret); + if (!br) { fprintf(stderr, "Buffer ring register failed %d\n", ret); return 1; } @@ -61,6 +56,7 @@ static int test_mixed_reg2(int bgid) } io_uring_cqe_seen(&ring, cqe); + io_uring_free_buf_ring(&ring, br, 32, bgid); io_uring_queue_exit(&ring); return 0; } @@ -68,11 +64,11 @@ static int test_mixed_reg2(int bgid) /* test trying to register ring group when classic group exists */ static int test_mixed_reg(int bgid) { - struct io_uring_buf_reg reg = { }; + struct io_uring_buf_ring *br; struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; struct io_uring ring; - void *ptr, *bufs; + void *bufs; int ret; ret = t_create_ring(1, &ring, 0); @@ -97,16 +93,9 @@ static int test_mixed_reg(int bgid) } io_uring_cqe_seen(&ring, cqe); - if (posix_memalign(&ptr, 4096, 4096)) - return 1; - - reg.ring_addr = (unsigned long) ptr; - reg.ring_entries = 32; - reg.bgid = bgid; - - ret = io_uring_register_buf_ring(&ring, ®, 0); - if (ret != -EEXIST) { - fprintf(stderr, "Buffer ring register failed %d\n", ret); + br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret); + if (br) { + fprintf(stderr, "Buffer ring setup succeeded unexpectedly %d\n", ret); return 1; } @@ -117,8 +106,8 @@ static int test_mixed_reg(int bgid) static int test_double_reg_unreg(int bgid) { struct io_uring_buf_reg reg = { }; + struct io_uring_buf_ring *br; struct io_uring ring; - void *ptr; int ret; ret = t_create_ring(1, &ring, 0); @@ -127,21 +116,14 @@ static int test_double_reg_unreg(int bgid) else if (ret != T_SETUP_OK) return 1; - if (posix_memalign(&ptr, 4096, 4096)) - return 1; - - reg.ring_addr = (unsigned long) ptr; - reg.ring_entries = 32; - reg.bgid = bgid; - - ret = io_uring_register_buf_ring(&ring, ®, 0); - if (ret) { + br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret); + if (!br) { fprintf(stderr, "Buffer ring register failed %d\n", ret); return 1; } /* check that 2nd register with same bgid fails */ - reg.ring_addr = (unsigned long) ptr; + reg.ring_addr = (unsigned long) br; reg.ring_entries = 32; reg.bgid = bgid; @@ -151,7 +133,7 @@ static int test_double_reg_unreg(int bgid) return 1; } - ret = io_uring_unregister_buf_ring(&ring, bgid); + ret = io_uring_free_buf_ring(&ring, br, 32, bgid); if (ret) { fprintf(stderr, "Buffer ring register failed %d\n", ret); return 1; @@ -169,9 +151,8 @@ static int test_double_reg_unreg(int bgid) static int test_reg_unreg(int bgid) { - struct io_uring_buf_reg reg = { }; + struct io_uring_buf_ring *br; struct io_uring ring; - void *ptr; int ret; ret = t_create_ring(1, &ring, 0); @@ -180,15 +161,8 @@ static int test_reg_unreg(int bgid) else if (ret != T_SETUP_OK) return 1; - if (posix_memalign(&ptr, 4096, 4096)) - return 1; - - reg.ring_addr = (unsigned long) ptr; - reg.ring_entries = 32; - reg.bgid = bgid; - - ret = io_uring_register_buf_ring(&ring, ®, 0); - if (ret) { + br = io_uring_setup_buf_ring(&ring, 32, bgid, 0, &ret); + if (!br) { if (ret == -EINVAL) { no_buf_ring = 1; return 0; @@ -197,9 +171,9 @@ static int test_reg_unreg(int bgid) return 1; } - ret = io_uring_unregister_buf_ring(&ring, bgid); + ret = io_uring_free_buf_ring(&ring, br, 32, bgid); if (ret) { - fprintf(stderr, "Buffer ring register failed %d\n", ret); + fprintf(stderr, "Buffer ring unregister failed %d\n", ret); return 1; } @@ -231,6 +205,54 @@ static int test_bad_reg(int bgid) return !ret; } +static int test_full_page_reg(int bgid) +{ +#if defined(__hppa__) + return T_EXIT_SKIP; +#else + struct io_uring ring; + int ret; + void *ptr; + struct io_uring_buf_reg reg = { }; + int entries = pagesize / sizeof(struct io_uring_buf); + + ret = io_uring_queue_init(1, &ring, 0); + if (ret) { + fprintf(stderr, "queue init failed %d\n", ret); + return T_EXIT_FAIL; + } + + ret = posix_memalign(&ptr, pagesize, pagesize * 2); + if (ret) { + fprintf(stderr, "posix_memalign failed %d\n", ret); + goto err; + } + + ret = mprotect(ptr + pagesize, pagesize, PROT_NONE); + if (ret) { + fprintf(stderr, "mprotect failed %d\n", errno); + goto err1; + } + + reg.ring_addr = (unsigned long) ptr; + reg.ring_entries = entries; + reg.bgid = bgid; + + ret = io_uring_register_buf_ring(&ring, ®, 0); + if (ret) + fprintf(stderr, "register buf ring failed %d\n", ret); + + if (mprotect(ptr + pagesize, pagesize, PROT_READ | PROT_WRITE)) + fprintf(stderr, "reverting mprotect failed %d\n", errno); + +err1: + free(ptr); +err: + io_uring_queue_exit(&ring); + return ret ? T_EXIT_FAIL : T_EXIT_PASS; +#endif +} + static int test_one_read(int fd, int bgid, struct io_uring *ring) { int ret; @@ -273,18 +295,12 @@ static int test_one_read(int fd, int bgid, struct io_uring *ring) static int test_running(int bgid, int entries, int loops) { - struct io_uring_buf_reg reg = { }; + int ring_mask = io_uring_buf_ring_mask(entries); + struct io_uring_buf_ring *br; + int ret, loop, idx, read_fd; struct io_uring ring; - void *ptr; char buffer[8]; - int ret; - int ring_size = (entries * sizeof(struct io_uring_buf) + 4095) & (~4095); - int ring_mask = io_uring_buf_ring_mask(entries); - - int loop, idx; bool *buffers; - struct io_uring_buf_ring *br; - int read_fd; ret = t_create_ring(1, &ring, 0); if (ret == T_SETUP_SKIP) @@ -292,11 +308,12 @@ static int test_running(int bgid, int entries, int loops) else if (ret != T_SETUP_OK) return 1; - if (posix_memalign(&ptr, 4096, ring_size)) + br = io_uring_setup_buf_ring(&ring, entries, bgid, 0, &ret); + if (!br) { + /* by now should have checked if this is supported or not */ + fprintf(stderr, "Buffer ring register failed %d\n", ret); return 1; - - br = (struct io_uring_buf_ring *)ptr; - io_uring_buf_ring_init(br); + } buffers = malloc(sizeof(bool) * entries); if (!buffers) @@ -306,17 +323,6 @@ static int test_running(int bgid, int entries, int loops) if (read_fd < 0) return 1; - reg.ring_addr = (unsigned long) ptr; - reg.ring_entries = entries; - reg.bgid = bgid; - - ret = io_uring_register_buf_ring(&ring, ®, 0); - if (ret) { - /* by now should have checked if this is supported or not */ - fprintf(stderr, "Buffer ring register failed %d\n", ret); - return 1; - } - for (loop = 0; loop < loops; loop++) { memset(buffers, 0, sizeof(bool) * entries); for (idx = 0; idx < entries; idx++) @@ -375,6 +381,8 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; + pagesize = getpagesize(); + for (i = 0; bgids[i] != -1; i++) { ret = test_reg_unreg(bgids[i]); if (ret) { @@ -407,6 +415,12 @@ int main(int argc, char *argv[]) fprintf(stderr, "test_mixed_reg2 failed\n"); return T_EXIT_FAIL; } + + ret = test_full_page_reg(bgids[i]); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test_full_page_reg failed\n"); + return T_EXIT_FAIL; + } } for (i = 0; !no_buf_ring && entries[i] != -1; i++) { diff --git a/contrib/libs/liburing/test/ce593a6c480a.c b/contrib/libs/liburing/test/ce593a6c480a.c index 7b6fc438ae..4677a095d4 100644 --- a/contrib/libs/liburing/test/ce593a6c480a.c +++ b/contrib/libs/liburing/test/ce593a6c480a.c @@ -16,7 +16,7 @@ static int use_sqpoll = 0; -void notify_fd(int fd) +static void notify_fd(int fd) { char buf[8] = {0, 0, 0, 0, 0, 0, 1}; int ret; @@ -26,7 +26,7 @@ void notify_fd(int fd) perror("write"); } -void *delay_set_fd_from_thread(void *data) +static void *delay_set_fd_from_thread(void *data) { int fd = (intptr_t) data; diff --git a/contrib/libs/liburing/test/close-opath.c b/contrib/libs/liburing/test/close-opath.c index 85fe30d9df..818493261d 100644 --- a/contrib/libs/liburing/test/close-opath.c +++ b/contrib/libs/liburing/test/close-opath.c @@ -16,12 +16,13 @@ #include <errno.h> #include <fcntl.h> -#include <liburing.h> #include <sys/stat.h> #include <stdio.h> #include <string.h> #include <unistd.h> +#include "liburing.h" + typedef struct { const char *const flnames; diff --git a/contrib/libs/liburing/test/connect-rep.c b/contrib/libs/liburing/test/connect-rep.c new file mode 100644 index 0000000000..ca300e7ac3 --- /dev/null +++ b/contrib/libs/liburing/test/connect-rep.c @@ -0,0 +1,205 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Check that repeated IORING_OP_CONNECT to a socket without a listener keeps + * yielding -ECONNREFUSED rather than -ECONNABORTED. Based on a reproducer + * from: + * + * https://github.com/axboe/liburing/issues/828 + * + * and adopted to our usual test cases. Other changes made like looping, + * using different ring types, adding a memset() for reuse, etc. + * + */ +#include <stdio.h> +#include <netinet/in.h> +#include <string.h> +#include <unistd.h> +#include <stdlib.h> +#include <arpa/inet.h> + +#include "liburing.h" +#include "helpers.h" + +static unsigned long ud; + +static int init_test_server(struct sockaddr_in *serv_addr) +{ + socklen_t servaddr_len = sizeof(struct sockaddr_in); + int fd; + + /* Init server socket. Bind but don't listen */ + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + perror("socket"); + return -1; + } + + serv_addr->sin_family = AF_INET; + serv_addr->sin_addr.s_addr = inet_addr("127.0.0.1"); + + if (bind(fd, (struct sockaddr *) serv_addr, servaddr_len) < 0) { + perror("bind"); + return -1; + } + + /* + * Get the addresses the socket is bound to because the port is chosen + * by the network stack. + */ + if (getsockname(fd, (struct sockaddr *)serv_addr, &servaddr_len) < 0) { + perror("getsockname"); + return -1; + } + + return fd; +} + +static int init_test_client(void) +{ + socklen_t addr_len = sizeof(struct sockaddr_in); + struct sockaddr_in client_addr = {}; + int clientfd; + + clientfd = socket(AF_INET, SOCK_STREAM, 0); + if (clientfd < 0) { + perror("socket"); + return -1; + } + + client_addr.sin_family = AF_INET; + client_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); + + if (bind(clientfd, (struct sockaddr *)&client_addr, addr_len) < 0) { + perror("bind"); + close(clientfd); + return -1; + } + + /* + * Get the addresses the socket is bound to because the port is chosen + * by the network stack. + */ + if (getsockname(clientfd, (struct sockaddr *)&client_addr, &addr_len) < 0) { + perror("getsockname"); + close(clientfd); + return -1; + } + + return clientfd; +} + +static int get_completion_and_print(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret, res; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return -1; + } + + /* Mark this completion as seen */ + res = cqe->res; + io_uring_cqe_seen(ring, cqe); + return res; +} + +static int test_connect(struct io_uring *ring, + int clientfd, struct sockaddr_in *serv_addr) +{ + struct sockaddr_in local_sa; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_connect(sqe, clientfd, (const struct sockaddr *)serv_addr, + sizeof(struct sockaddr_in)); + sqe->user_data = ++ud; + + memcpy(&local_sa, serv_addr, sizeof(local_sa)); + + ret = io_uring_submit_and_wait(ring, 1); + if (ret != 1) { + fprintf(stderr, "submit=%d\n", ret); + return T_EXIT_FAIL; + } + + /* check for reuse at the same time */ + memset(&local_sa, 0xff, sizeof(local_sa)); + + ret = get_completion_and_print(ring); + if (ret != -ECONNREFUSED) { + fprintf(stderr, "Connect got %d\n", ret); + return T_EXIT_FAIL; + } + return T_EXIT_PASS; +} + +static int test(int flags) +{ + struct io_uring_params params = { .flags = flags, }; + struct sockaddr_in serv_addr = {}; + struct io_uring ring; + int ret, clientfd, s_fd, i; + + if (flags & IORING_SETUP_SQPOLL) + params.sq_thread_idle = 50; + + ret = io_uring_queue_init_params(8, &ring, ¶ms); + if (ret < 0) { + fprintf(stderr, "Queue init: %d\n", ret); + return T_EXIT_FAIL; + } + + s_fd = init_test_server(&serv_addr); + if (s_fd < 0) + return T_EXIT_FAIL; + + clientfd = init_test_client(); + if (clientfd < 0) { + close(s_fd); + return T_EXIT_FAIL; + } + + /* make sure SQPOLL thread is sleeping */ + if (flags & IORING_SETUP_SQPOLL) + usleep(100000); + + for (i = 0; i < 32; i++) { + ret = test_connect(&ring, clientfd, &serv_addr); + if (ret == T_EXIT_SKIP) + return T_EXIT_SKIP; + else if (ret == T_EXIT_PASS) + continue; + + return T_EXIT_FAIL; + } + + close(s_fd); + close(clientfd); + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = test(0); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test(0) failed\n"); + return T_EXIT_FAIL; + } + + ret = test(IORING_SETUP_SQPOLL); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test(SQPOLL) failed\n"); + return T_EXIT_FAIL; + } + + return 0; +} diff --git a/contrib/libs/liburing/test/timeout-overflow.t/ya.make b/contrib/libs/liburing/test/connect-rep.t/ya.make index de0ef396f3..556843fd8b 100644 --- a/contrib/libs/liburing/test/timeout-overflow.t/ya.make +++ b/contrib/libs/liburing/test/connect-rep.t/ya.make @@ -24,8 +24,8 @@ CFLAGS( SRCDIR(contrib/libs/liburing/test) SRCS( + connect-rep.c helpers.c - timeout-overflow.c ) END() diff --git a/contrib/libs/liburing/test/connect.c b/contrib/libs/liburing/test/connect.c index 30f3ac1c3f..f1b7d941f4 100644 --- a/contrib/libs/liburing/test/connect.c +++ b/contrib/libs/liburing/test/connect.c @@ -16,6 +16,7 @@ #include <netinet/in.h> #include <netinet/tcp.h> #include <arpa/inet.h> +#include <sys/stat.h> #include "liburing.h" #include "helpers.h" @@ -257,6 +258,13 @@ static int test_connect_timeout(struct io_uring *ring) struct sockaddr_in addr; struct io_uring_sqe *sqe; struct __kernel_timespec ts = {.tv_sec = 0, .tv_nsec = 100000}; + struct stat sb; + + /* + * Test reliably fails if syncookies isn't enabled + */ + if (stat("/proc/sys/net/ipv4/tcp_syncookies", &sb) < 0) + return T_EXIT_SKIP; connect_fd[0] = create_socket(); if (connect_fd[0] == -1) diff --git a/contrib/libs/liburing/test/cq-overflow.c b/contrib/libs/liburing/test/cq-overflow.c index f43e19c013..569c733752 100644 --- a/contrib/libs/liburing/test/cq-overflow.c +++ b/contrib/libs/liburing/test/cq-overflow.c @@ -49,7 +49,8 @@ static struct iovec *vecs; * bash -c "echo 1 > /proc/self/make-it-fail && exec ./cq-overflow.t" */ -static int test_io(const char *file, unsigned long usecs, unsigned *drops, int fault) +static int test_io(const char *file, unsigned long usecs, unsigned *drops, + int fault) { struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; @@ -61,8 +62,10 @@ static int test_io(const char *file, unsigned long usecs, unsigned *drops, int f fd = open(file, O_RDONLY | O_DIRECT); if (fd < 0) { + if (errno == EINVAL) + return T_EXIT_SKIP; perror("file open"); - return 1; + return T_EXIT_FAIL; } memset(&p, 0, sizeof(p)); @@ -70,7 +73,7 @@ static int test_io(const char *file, unsigned long usecs, unsigned *drops, int f if (ret) { close(fd); fprintf(stderr, "ring create failed: %d\n", ret); - return 1; + return T_EXIT_FAIL; } nodrop = 0; if (p.features & IORING_FEAT_NODROP) @@ -174,12 +177,12 @@ reap_it: io_uring_queue_exit(&ring); close(fd); - return 0; + return T_EXIT_PASS; err: if (fd != -1) close(fd); io_uring_queue_exit(&ring); - return 1; + return T_EXIT_SKIP; } static int reap_events(struct io_uring *ring, unsigned nr_events, int do_wait) @@ -497,7 +500,10 @@ int main(int argc, char *argv[]) do { drops = 0; - if (test_io(fname, usecs, &drops, 0)) { + ret = test_io(fname, usecs, &drops, 0); + if (ret == T_EXIT_SKIP) + break; + else if (ret != T_EXIT_PASS) { fprintf(stderr, "test_io nofault failed\n"); goto err; } @@ -507,12 +513,12 @@ int main(int argc, char *argv[]) iters++; } while (iters < 40); - if (test_io(fname, usecs, &drops, 0)) { + if (test_io(fname, usecs, &drops, 0) == T_EXIT_FAIL) { fprintf(stderr, "test_io nofault failed\n"); goto err; } - if (test_io(fname, usecs, &drops, 1)) { + if (test_io(fname, usecs, &drops, 1) == T_EXIT_FAIL) { fprintf(stderr, "test_io fault failed\n"); goto err; } diff --git a/contrib/libs/liburing/test/d4ae271dfaae.c b/contrib/libs/liburing/test/d4ae271dfaae.c index 1a7886ed6b..af47047be8 100644 --- a/contrib/libs/liburing/test/d4ae271dfaae.c +++ b/contrib/libs/liburing/test/d4ae271dfaae.c @@ -30,7 +30,7 @@ int main(int argc, char *argv[]) memset(&p, 0, sizeof(p)); p.flags = IORING_SETUP_SQPOLL; - ret = t_create_ring_params(4, &ring, &p); + ret = t_create_ring_params(16, &ring, &p); if (ret == T_SETUP_SKIP) return T_EXIT_SKIP; else if (ret < 0) diff --git a/contrib/libs/liburing/test/defer-taskrun.c b/contrib/libs/liburing/test/defer-taskrun.c index b07ac2f778..c2543f4b26 100644 --- a/contrib/libs/liburing/test/defer-taskrun.c +++ b/contrib/libs/liburing/test/defer-taskrun.c @@ -5,7 +5,6 @@ #include <unistd.h> #include <stdlib.h> #include <string.h> -#include <error.h> #include <sys/eventfd.h> #include <signal.h> #include <poll.h> @@ -58,9 +57,14 @@ static void eventfd_trigger(int fd) assert(ret == sizeof(val)); } -#define CHECK(x) if (!(x)) { \ - fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \ - return -1; } +#define CHECK(x) \ +do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \ + return -1; \ + } \ +} while (0) + static int test_eventfd(void) { @@ -120,7 +124,7 @@ struct thread_data { char buff[8]; }; -void *thread(void *t) +static void *thread(void *t) { struct thread_data *td = t; @@ -179,11 +183,11 @@ static int test_exec(const char *filename) int wstatus; CHECK(waitpid(fork_pid, &wstatus, 0) != (pid_t)-1); - if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) != T_EXIT_SKIP) { + if (!WIFEXITED(wstatus) || WEXITSTATUS(wstatus) == T_EXIT_FAIL) { fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus)); return -1; } - return 0; + return T_EXIT_PASS; } ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | @@ -193,9 +197,15 @@ static int test_exec(const char *filename) if (filename) { fd = open(filename, O_RDONLY | O_DIRECT); + if (fd < 0 && errno == EINVAL) + return T_EXIT_SKIP; } else { t_create_file(EXEC_FILENAME, EXEC_FILESIZE); fd = open(EXEC_FILENAME, O_RDONLY | O_DIRECT); + if (fd < 0 && errno == EINVAL) { + unlink(EXEC_FILENAME); + return T_EXIT_SKIP; + } unlink(EXEC_FILENAME); } buff = (char*)malloc(EXEC_FILESIZE); @@ -208,7 +218,7 @@ static int test_exec(const char *filename) ret = execve("/proc/self/exe", new_argv, new_env); /* if we get here it failed anyway */ fprintf(stderr, "execve failed %d\n", ret); - return -1; + return T_EXIT_FAIL; } static int test_flag(void) @@ -284,6 +294,45 @@ static int test_ring_shutdown(void) return 0; } +static int test_drain(void) +{ + struct io_uring ring; + int ret, i, fd[2]; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct iovec iovecs[128]; + char buff[ARRAY_SIZE(iovecs)]; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN | + IORING_SETUP_TASKRUN_FLAG); + CHECK(!ret); + + for (i = 0; i < ARRAY_SIZE(iovecs); i++) { + iovecs[i].iov_base = &buff[i]; + iovecs[i].iov_len = 1; + } + + ret = t_create_socket_pair(fd, true); + CHECK(!ret); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_writev(sqe, fd[1], &iovecs[0], ARRAY_SIZE(iovecs), 0); + sqe->flags |= IOSQE_IO_DRAIN; + io_uring_submit(&ring); + + for (i = 0; i < ARRAY_SIZE(iovecs); i++) + iovecs[i].iov_base = NULL; + + CHECK(io_uring_wait_cqe(&ring, &cqe) == 0); + CHECK(cqe->res == 128); + + close(fd[0]); + close(fd[1]); + io_uring_queue_exit(&ring); + return 0; +} + int main(int argc, char *argv[]) { int ret; @@ -310,7 +359,7 @@ int main(int argc, char *argv[]) } ret = test_exec(filename); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_exec failed\n"); return T_EXIT_FAIL; } @@ -333,5 +382,11 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } + ret = test_drain(); + if (ret) { + fprintf(stderr, "test_drain failed\n"); + return T_EXIT_FAIL; + } + return T_EXIT_PASS; } diff --git a/contrib/libs/liburing/test/defer.c b/contrib/libs/liburing/test/defer.c index 4e7cd23135..c1e7478906 100644 --- a/contrib/libs/liburing/test/defer.c +++ b/contrib/libs/liburing/test/defer.c @@ -58,7 +58,7 @@ static int init_context(struct test_context *ctx, struct io_uring *ring, int nr, case OP_REMOVE_BUFFERS: io_uring_prep_remove_buffers(sqe, 10, 1); break; - }; + } sqe->user_data = i; ctx->sqes[i] = sqe; } diff --git a/contrib/libs/liburing/test/double-poll-crash.c b/contrib/libs/liburing/test/double-poll-crash.c index c24ad86661..67be0682c4 100644 --- a/contrib/libs/liburing/test/double-poll-crash.c +++ b/contrib/libs/liburing/test/double-poll-crash.c @@ -110,7 +110,7 @@ static long syz_open_dev(volatile long a0, volatile long a1, volatile long a2) } } -uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff}; +static uint64_t r[4] = {0xffffffffffffffff, 0x0, 0x0, 0xffffffffffffffff}; int main(int argc, char *argv[]) { @@ -122,10 +122,10 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; - mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + mmap_ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); if (mmap_ret == MAP_FAILED) return T_EXIT_SKIP; - mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + mmap_ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); if (mmap_ret == MAP_FAILED) return T_EXIT_SKIP; intptr_t res = 0; diff --git a/contrib/libs/liburing/test/eeed8b54e0df.c b/contrib/libs/liburing/test/eeed8b54e0df.c index c4118a2d51..6b73a35db6 100644 --- a/contrib/libs/liburing/test/eeed8b54e0df.c +++ b/contrib/libs/liburing/test/eeed8b54e0df.c @@ -103,13 +103,18 @@ int main(int argc, char *argv[]) goto err; } + ret = T_EXIT_PASS; if (cqe->res != -EAGAIN && cqe->res != 4096) { - printf("cqe error: %d\n", cqe->res); - goto err; + if (cqe->res == -EOPNOTSUPP) { + ret = T_EXIT_SKIP; + } else { + printf("cqe error: %d\n", cqe->res); + goto err; + } } close(fd); - return T_EXIT_PASS; + return ret; err: close(fd); return T_EXIT_FAIL; diff --git a/contrib/libs/liburing/test/eploop.c b/contrib/libs/liburing/test/eploop.c new file mode 100644 index 0000000000..f3fd3f582e --- /dev/null +++ b/contrib/libs/liburing/test/eploop.c @@ -0,0 +1,75 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test that we don't recursively generate completion events if an io_uring + * fd is added to an epoll context, and the ring itself polls for events on + * the epollfd. Older kernels will stop on overflow, newer kernels will + * detect this earlier and abort correctly. + */ +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <sys/epoll.h> +#include <sys/types.h> +#include <poll.h> +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct epoll_event ev = { }; + int epollfd, ret, i; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "Ring init failed: %d\n", ret); + return T_EXIT_FAIL; + } + + epollfd = epoll_create1(0); + if (epollfd < 0) { + perror("epoll_create"); + return T_EXIT_FAIL; + } + + ev.events = EPOLLIN; + ev.data.fd = ring.ring_fd; + ret = epoll_ctl(epollfd, EPOLL_CTL_ADD, ring.ring_fd, &ev); + if (ret < 0) { + perror("epoll_ctl"); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_multishot(sqe, epollfd, POLLIN); + sqe->user_data = 1; + io_uring_submit(&ring); + + sqe = io_uring_get_sqe(&ring); + sqe->user_data = 2; + io_uring_prep_nop(sqe); + io_uring_submit(&ring); + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe ret = %d\n", ret); + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + ret = io_uring_peek_cqe(&ring, &cqe); + if (!ret) { + fprintf(stderr, "Generated too many events\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/eploop.t/ya.make b/contrib/libs/liburing/test/eploop.t/ya.make new file mode 100644 index 0000000000..29b1f9096a --- /dev/null +++ b/contrib/libs/liburing/test/eploop.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + eploop.c + helpers.c +) + +END() diff --git a/contrib/libs/liburing/test/eventfd-ring.c b/contrib/libs/liburing/test/eventfd-ring.c index d4bed86526..e0a3eecaa5 100644 --- a/contrib/libs/liburing/test/eventfd-ring.c +++ b/contrib/libs/liburing/test/eventfd-ring.c @@ -1,7 +1,7 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ /* - * Description: run various nop tests + * Description: test use of eventfds with multiple rings * */ #include <errno.h> diff --git a/contrib/libs/liburing/test/eventfd.c b/contrib/libs/liburing/test/eventfd.c index 317de29fb4..081987f520 100644 --- a/contrib/libs/liburing/test/eventfd.c +++ b/contrib/libs/liburing/test/eventfd.c @@ -1,7 +1,7 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ /* - * Description: run various nop tests + * Description: run various eventfd tests * */ #include <errno.h> diff --git a/contrib/libs/liburing/test/evloop.c b/contrib/libs/liburing/test/evloop.c new file mode 100644 index 0000000000..a4339d9471 --- /dev/null +++ b/contrib/libs/liburing/test/evloop.c @@ -0,0 +1,74 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test that we don't recursively generate completion events if an io_uring + * has an eventfd registered that triggers on completions, and we add a poll + * request with multishot on the eventfd. Older kernels will stop on overflow, + * newer kernels will detect this earlier and abort correctly. + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <sys/eventfd.h> +#include <sys/types.h> +#include <poll.h> +#include <assert.h> +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int ret, efd, i; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "Ring init failed: %d\n", ret); + return T_EXIT_FAIL; + } + + efd = eventfd(0, 0); + if (efd < 0) { + perror("eventfd"); + return T_EXIT_FAIL; + } + + ret = io_uring_register_eventfd(&ring, efd); + if (ret) { + fprintf(stderr, "Ring eventfd register failed: %d\n", ret); + return T_EXIT_FAIL; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_multishot(sqe, efd, POLLIN); + sqe->user_data = 1; + io_uring_submit(&ring); + + sqe = io_uring_get_sqe(&ring); + sqe->user_data = 2; + io_uring_prep_nop(sqe); + io_uring_submit(&ring); + + for (i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe ret = %d\n", ret); + break; + } + io_uring_cqe_seen(&ring, cqe); + } + + ret = io_uring_peek_cqe(&ring, &cqe); + if (!ret) { + fprintf(stderr, "Generated too many events\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/evloop.t/ya.make b/contrib/libs/liburing/test/evloop.t/ya.make new file mode 100644 index 0000000000..6040603610 --- /dev/null +++ b/contrib/libs/liburing/test/evloop.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + evloop.c + helpers.c +) + +END() diff --git a/contrib/libs/liburing/test/exit-no-cleanup.c b/contrib/libs/liburing/test/exit-no-cleanup.c index 2fe3dc7ddd..abf63ab2b0 100644 --- a/contrib/libs/liburing/test/exit-no-cleanup.c +++ b/contrib/libs/liburing/test/exit-no-cleanup.c @@ -27,7 +27,7 @@ static pthread_barrier_t init_barrier; static int sleep_fd, notify_fd; static sem_t sem; -void *thread_func(void *arg) +static void *thread_func(void *arg) { struct io_uring ring; int res; diff --git a/contrib/libs/liburing/test/fadvise.c b/contrib/libs/liburing/test/fadvise.c index 86670b80f4..12b7ffd912 100644 --- a/contrib/libs/liburing/test/fadvise.c +++ b/contrib/libs/liburing/test/fadvise.c @@ -187,7 +187,7 @@ int main(int argc, char *argv[]) } /* too hard to reliably test, just ignore */ - if (0 && bad > good) { + if ((0) && bad > good) { fprintf(stderr, "Suspicious timings\n"); goto err; } diff --git a/contrib/libs/liburing/test/fallocate.c b/contrib/libs/liburing/test/fallocate.c index a546922f96..cff01528fc 100644 --- a/contrib/libs/liburing/test/fallocate.c +++ b/contrib/libs/liburing/test/fallocate.c @@ -13,6 +13,7 @@ #include <stdlib.h> #include <string.h> #include <fcntl.h> +#include <signal.h> #include "liburing.h" #include "helpers.h" @@ -217,14 +218,22 @@ err: return 1; } +static void sig_xfsz(int sig) +{ +} + int main(int argc, char *argv[]) { + struct sigaction act = { }; struct io_uring ring; int ret; if (argc > 1) return T_EXIT_SKIP; + act.sa_handler = sig_xfsz; + sigaction(SIGXFSZ, &act, NULL); + ret = io_uring_queue_init(8, &ring, 0); if (ret) { fprintf(stderr, "ring setup failed\n"); diff --git a/contrib/libs/liburing/test/fc2a85cb02ef.c b/contrib/libs/liburing/test/fc2a85cb02ef.c index 3c61374845..538e8d4312 100644 --- a/contrib/libs/liburing/test/fc2a85cb02ef.c +++ b/contrib/libs/liburing/test/fc2a85cb02ef.c @@ -55,7 +55,7 @@ static int inject_fault(int nth) return fd; } -static int setup_fault() +static int setup_fault(void) { static struct { const char* file; @@ -80,13 +80,13 @@ static int setup_fault() return 0; } -uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff}; +static uint64_t r[2] = {0xffffffffffffffff, 0xffffffffffffffff}; int main(int argc, char *argv[]) { if (argc > 1) return T_EXIT_SKIP; - mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, 0x32ul, -1, 0); + mmap((void *) 0x20000000ul, 0x1000000ul, 3ul, MAP_ANON|MAP_PRIVATE, -1, 0); if (setup_fault()) { printf("Test needs failslab/fail_futex/fail_page_alloc enabled, skipped\n"); return T_EXIT_SKIP; diff --git a/contrib/libs/liburing/test/fd-pass.c b/contrib/libs/liburing/test/fd-pass.c index 9549d418e5..c5da83592b 100644 --- a/contrib/libs/liburing/test/fd-pass.c +++ b/contrib/libs/liburing/test/fd-pass.c @@ -14,8 +14,9 @@ #include "liburing.h" #include "helpers.h" -#define FSIZE 128 -#define PAT 0x9a +#define FSIZE 128 +#define PAT 0x9a +#define USER_DATA 0x89 static int no_fd_pass; @@ -50,7 +51,7 @@ static int verify_fixed_read(struct io_uring *ring, int fixed_fd, int fail) return 0; } -static int test(const char *filename) +static int test(const char *filename, int source_fd, int target_fd) { struct io_uring sring, dring; struct io_uring_sqe *sqe; @@ -80,10 +81,18 @@ static int test(const char *filename) fprintf(stderr, "register files failed %d\n", ret); return T_EXIT_FAIL; } + if (target_fd == IORING_FILE_INDEX_ALLOC) { + /* we want to test installing into a non-zero slot */ + ret = io_uring_register_file_alloc_range(&dring, 1, 1); + if (ret) { + fprintf(stderr, "io_uring_register_file_alloc_range %d\n", ret); + return T_EXIT_FAIL; + } + } /* open direct descriptor */ sqe = io_uring_get_sqe(&sring); - io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, 0); + io_uring_prep_openat_direct(sqe, AT_FDCWD, filename, 0, 0644, source_fd); io_uring_submit(&sring); ret = io_uring_wait_cqe(&sring, &cqe); if (ret) { @@ -97,15 +106,19 @@ static int test(const char *filename) io_uring_cqe_seen(&sring, cqe); /* verify data is sane for source ring */ - if (verify_fixed_read(&sring, 0, 0)) + if (verify_fixed_read(&sring, source_fd, 0)) return T_EXIT_FAIL; /* send direct descriptor to destination ring */ sqe = io_uring_get_sqe(&sring); - io_uring_prep_msg_ring(sqe, dring.ring_fd, 0, 0x89, 0); - sqe->addr = 1; - sqe->addr3 = 0; - sqe->file_index = 1; + if (target_fd == IORING_FILE_INDEX_ALLOC) { + io_uring_prep_msg_ring_fd_alloc(sqe, dring.ring_fd, source_fd, + USER_DATA, 0); + } else { + + io_uring_prep_msg_ring_fd(sqe, dring.ring_fd, source_fd, + target_fd, USER_DATA, 0); + } io_uring_submit(&sring); ret = io_uring_wait_cqe(&sring, &cqe); @@ -113,7 +126,7 @@ static int test(const char *filename) fprintf(stderr, "wait cqe failed %d\n", ret); return T_EXIT_FAIL; } - if (cqe->res) { + if (cqe->res < 0) { if (cqe->res == -EINVAL && !no_fd_pass) { no_fd_pass = 1; return T_EXIT_SKIP; @@ -129,19 +142,30 @@ static int test(const char *filename) fprintf(stderr, "wait cqe failed %d\n", ret); return T_EXIT_FAIL; } - if (cqe->user_data != 0x89) { + if (cqe->user_data != USER_DATA) { fprintf(stderr, "bad user_data %ld\n", (long) cqe->res); return T_EXIT_FAIL; } + if (cqe->res < 0) { + fprintf(stderr, "bad result %i\n", cqe->res); + return T_EXIT_FAIL; + } + if (target_fd == IORING_FILE_INDEX_ALLOC) { + if (cqe->res != 1) { + fprintf(stderr, "invalid allocated index %i\n", cqe->res); + return T_EXIT_FAIL; + } + target_fd = cqe->res; + } io_uring_cqe_seen(&dring, cqe); /* now verify we can read the sane data from the destination ring */ - if (verify_fixed_read(&dring, 0, 0)) + if (verify_fixed_read(&dring, target_fd, 0)) return T_EXIT_FAIL; /* close descriptor in source ring */ sqe = io_uring_get_sqe(&sring); - io_uring_prep_close_direct(sqe, 0); + io_uring_prep_close_direct(sqe, source_fd); io_uring_submit(&sring); ret = io_uring_wait_cqe(&sring, &cqe); @@ -156,13 +180,15 @@ static int test(const char *filename) io_uring_cqe_seen(&sring, cqe); /* check that source ring fails after close */ - if (verify_fixed_read(&sring, 0, 1)) + if (verify_fixed_read(&sring, source_fd, 1)) return T_EXIT_FAIL; /* check we can still read from destination ring */ - if (verify_fixed_read(&dring, 0, 0)) + if (verify_fixed_read(&dring, target_fd, 0)) return T_EXIT_FAIL; + io_uring_queue_exit(&sring); + io_uring_queue_exit(&dring); return T_EXIT_PASS; } @@ -177,9 +203,33 @@ int main(int argc, char *argv[]) sprintf(fname, ".fd-pass.%d", getpid()); t_create_file_pattern(fname, FSIZE, PAT); - ret = test(fname); + ret = test(fname, 0, 1); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test failed 0 1\n"); + ret = T_EXIT_FAIL; + } + + ret = test(fname, 0, 2); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test failed 0 2\n"); + ret = T_EXIT_FAIL; + } + + ret = test(fname, 1, 1); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test failed 1 1\n"); + ret = T_EXIT_FAIL; + } + + ret = test(fname, 1, 0); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test failed 1 0\n"); + ret = T_EXIT_FAIL; + } + + ret = test(fname, 1, IORING_FILE_INDEX_ALLOC); if (ret == T_EXIT_FAIL) { - fprintf(stderr, "test failed\n"); + fprintf(stderr, "test failed 1 ALLOC\n"); ret = T_EXIT_FAIL; } diff --git a/contrib/libs/liburing/test/file-register.c b/contrib/libs/liburing/test/file-register.c index 707cd5422f..ee30b3e4f8 100644 --- a/contrib/libs/liburing/test/file-register.c +++ b/contrib/libs/liburing/test/file-register.c @@ -936,6 +936,59 @@ static int test_zero_range_alloc(struct io_uring *ring, int fds[2]) return 0; } +static int test_defer_taskrun(void) +{ + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret, fds[2]; + char buff = 'x'; + + ret = io_uring_queue_init(8, &ring, + IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER); + if (ret) { + fprintf(stderr, "ring init\n"); + return 1; + } + + ret = pipe(fds); + if (ret) { + fprintf(stderr, "bad pipes\n"); + return 1; + } + + ret = io_uring_register_files(&ring, &fds[0], 2); + if (ret) { + fprintf(stderr, "bad register %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_read(sqe, 0, &buff, 1, 0); + sqe->flags |= IOSQE_FIXED_FILE; + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "bad submit\n"); + return 1; + } + + ret = write(fds[1], &buff, 1); + if (ret != 1) { + fprintf(stderr, "bad pipe write\n"); + return 1; + } + + ret = io_uring_unregister_files(&ring); + if (ret) { + fprintf(stderr, "bad unregister %d\n", ret); + return 1; + } + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + static int test_file_alloc_ranges(void) { struct io_uring ring; @@ -1121,5 +1174,13 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } + if (t_probe_defer_taskrun()) { + ret = test_defer_taskrun(); + if (ret) { + fprintf(stderr, "test_defer_taskrun failed\n"); + return T_EXIT_FAIL; + } + } + return T_EXIT_PASS; } diff --git a/contrib/libs/liburing/test/file-verify.c b/contrib/libs/liburing/test/file-verify.c index 7950b739cc..fd192ec272 100644 --- a/contrib/libs/liburing/test/file-verify.c +++ b/contrib/libs/liburing/test/file-verify.c @@ -11,7 +11,6 @@ #include <string.h> #include <fcntl.h> #include <assert.h> -#include <string.h> #include <sys/ioctl.h> #include <sys/stat.h> #include <linux/fs.h> @@ -34,20 +33,38 @@ */ #define READ_BATCH 16 +static void verify_buf_sync(void *buf, size_t size, bool registered) +{ +#if defined(__hppa__) + if (registered) { + unsigned long off = (unsigned long) buf & 4095; + unsigned long p = (unsigned long) buf & ~4095; + int i; + + size += off; + for (i = 0; i < size; i += 32) + asm volatile("fdc 0(%0)" : : "r" (p + i)); + } +#endif +} + /* * Each offset in the file has the offset / sizeof(int) stored for every * sizeof(int) address. */ -static int verify_buf(void *buf, size_t size, off_t off) +static int verify_buf(void *buf, size_t size, off_t off, bool registered) { int i, u_in_buf = size / sizeof(unsigned int); unsigned int *ptr; + verify_buf_sync(buf, size, registered); + off /= sizeof(unsigned int); ptr = buf; for (i = 0; i < u_in_buf; i++) { if (off != *ptr) { - fprintf(stderr, "Found %u, wanted %lu\n", *ptr, off); + fprintf(stderr, "Found %u, wanted %llu\n", *ptr, + (unsigned long long) off); return 1; } ptr++; @@ -198,7 +215,7 @@ again: goto err; } - if (verify_buf(buf, CHUNK_SIZE / 2, 0)) + if (verify_buf(buf, CHUNK_SIZE / 2, 0, false)) goto err; /* @@ -365,9 +382,12 @@ static int test(struct io_uring *ring, const char *fname, int buffered, v[i].iov_base = buf[i]; v[i].iov_len = CHUNK_SIZE; } - ret = io_uring_register_buffers(ring, v, READ_BATCH); + ret = t_register_buffers(ring, v, READ_BATCH); if (ret) { - fprintf(stderr, "Error buffer reg %d\n", ret); + if (ret == T_SETUP_SKIP) { + ret = 0; + goto free_bufs; + } goto err; } } @@ -446,12 +466,12 @@ static int test(struct io_uring *ring, const char *fname, int buffered, void *buf = vecs[index][j].iov_base; size_t len = vecs[index][j].iov_len; - if (verify_buf(buf, len, voff)) + if (verify_buf(buf, len, voff, registered)) goto err; voff += len; } } else { - if (verify_buf(buf[index], CHUNK_SIZE, voff)) + if (verify_buf(buf[index], CHUNK_SIZE, voff, registered)) goto err; } } @@ -461,6 +481,7 @@ static int test(struct io_uring *ring, const char *fname, int buffered, done: if (registered) io_uring_unregister_buffers(ring); +free_bufs: if (vectored) { for (j = 0; j < READ_BATCH; j++) for (i = 0; i < nr_vecs; i++) diff --git a/contrib/libs/liburing/test/files-exit-hang-timeout.c b/contrib/libs/liburing/test/files-exit-hang-timeout.c index 708e42cded..a549e92cdb 100644 --- a/contrib/libs/liburing/test/files-exit-hang-timeout.c +++ b/contrib/libs/liburing/test/files-exit-hang-timeout.c @@ -22,9 +22,9 @@ #define PORT 9100 -struct io_uring ring; +static struct io_uring ring; -struct __kernel_timespec ts = { +static struct __kernel_timespec ts = { .tv_sec = 300, .tv_nsec = 0, }; diff --git a/contrib/libs/liburing/test/fixed-link.c b/contrib/libs/liburing/test/fixed-link.c index 3e9eb8c4f1..d37788af03 100644 --- a/contrib/libs/liburing/test/fixed-link.c +++ b/contrib/libs/liburing/test/fixed-link.c @@ -37,7 +37,7 @@ int main(int argc, char *argv[]) for (i = 0; i < IOVECS_LEN; ++i) { iovecs[i].iov_base = t_malloc(64); iovecs[i].iov_len = 64; - }; + } ret = io_uring_register_buffers(&ring, iovecs, IOVECS_LEN); if (ret) { diff --git a/contrib/libs/liburing/test/hardlink.c b/contrib/libs/liburing/test/hardlink.c index 1d85fe0046..1bd7a1c882 100644 --- a/contrib/libs/liburing/test/hardlink.c +++ b/contrib/libs/liburing/test/hardlink.c @@ -45,7 +45,7 @@ err: return 1; } -int files_linked_ok(const char* fn1, const char *fn2) +static int files_linked_ok(const char* fn1, const char *fn2) { struct stat s1, s2; diff --git a/contrib/libs/liburing/test/helpers.c b/contrib/libs/liburing/test/helpers.c index a29c7b5966..7133b00b91 100644 --- a/contrib/libs/liburing/test/helpers.c +++ b/contrib/libs/liburing/test/helpers.c @@ -9,6 +9,7 @@ #include <stdio.h> #include <fcntl.h> #include <unistd.h> +#include <stdarg.h> #include <sys/types.h> #include <arpa/inet.h> @@ -135,7 +136,8 @@ enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring, return T_SETUP_SKIP; } - fprintf(stderr, "queue_init: %s\n", strerror(-ret)); + if (ret != -EINVAL) + fprintf(stderr, "queue_init: %s\n", strerror(-ret)); return ret; } @@ -174,7 +176,7 @@ int t_create_socket_pair(int fd[2], bool stream) int val; struct sockaddr_in serv_addr; struct sockaddr *paddr; - size_t paddrlen; + socklen_t paddrlen; type |= SOCK_CLOEXEC; fd[0] = socket(AF_INET, type, 0); @@ -267,3 +269,53 @@ bool t_probe_defer_taskrun(void) io_uring_queue_exit(&ring); return true; } + +/* + * Sync internal state with kernel ring state on the SQ side. Returns the + * number of pending items in the SQ ring, for the shared ring. + */ +unsigned __io_uring_flush_sq(struct io_uring *ring) +{ + struct io_uring_sq *sq = &ring->sq; + unsigned tail = sq->sqe_tail; + + if (sq->sqe_head != tail) { + sq->sqe_head = tail; + /* + * Ensure kernel sees the SQE updates before the tail update. + */ + if (!(ring->flags & IORING_SETUP_SQPOLL)) + IO_URING_WRITE_ONCE(*sq->ktail, tail); + else + io_uring_smp_store_release(sq->ktail, tail); + } + /* + * This _may_ look problematic, as we're not supposed to be reading + * SQ->head without acquire semantics. When we're in SQPOLL mode, the + * kernel submitter could be updating this right now. For non-SQPOLL, + * task itself does it, and there's no potential race. But even for + * SQPOLL, the load is going to be potentially out-of-date the very + * instant it's done, regardless or whether or not it's done + * atomically. Worst case, we're going to be over-estimating what + * we can submit. The point is, we need to be able to deal with this + * situation regardless of any perceived atomicity. + */ + return tail - *sq->khead; +} + +/* + * Implementation of error(3), prints an error message and exits. + */ +void t_error(int status, int errnum, const char *format, ...) +{ + va_list args; + va_start(args, format); + + vfprintf(stderr, format, args); + if (errnum) + fprintf(stderr, ": %s", strerror(errnum)); + + fprintf(stderr, "\n"); + va_end(args); + exit(status); +} diff --git a/contrib/libs/liburing/test/helpers.h b/contrib/libs/liburing/test/helpers.h index 4375a9e465..530732422c 100644 --- a/contrib/libs/liburing/test/helpers.h +++ b/contrib/libs/liburing/test/helpers.h @@ -85,8 +85,12 @@ enum t_setup_ret t_register_buffers(struct io_uring *ring, bool t_probe_defer_taskrun(void); +unsigned __io_uring_flush_sq(struct io_uring *ring); + #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +void t_error(int status, int errnum, const char *format, ...); + #ifdef __cplusplus } #endif diff --git a/contrib/libs/liburing/test/io-cancel.c b/contrib/libs/liburing/test/io-cancel.c index 59932ff9a4..8d769ea704 100644 --- a/contrib/libs/liburing/test/io-cancel.c +++ b/contrib/libs/liburing/test/io-cancel.c @@ -162,6 +162,8 @@ static int test_io_cancel(const char *file, int do_write, int do_partial, fd = open(file, O_RDWR | O_DIRECT); if (fd < 0) { + if (errno == EINVAL) + return T_EXIT_SKIP; perror("file open"); goto err; } @@ -541,7 +543,7 @@ int main(int argc, char *argv[]) int async = (i & 4) != 0; ret = test_io_cancel(fname, write, partial, async); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "test_io_cancel %d %d %d failed\n", write, partial, async); goto err; diff --git a/contrib/libs/liburing/test/io_uring_passthrough.c b/contrib/libs/liburing/test/io_uring_passthrough.c index 156347456e..ad6b6d3e5f 100644 --- a/contrib/libs/liburing/test/io_uring_passthrough.c +++ b/contrib/libs/liburing/test/io_uring_passthrough.c @@ -19,6 +19,7 @@ #define BUFFERS (FILE_SIZE / BS) static struct iovec *vecs; +static int no_pt; /* * Each offset in the file has the ((test_case / 2) * FILE_SIZE) @@ -35,7 +36,8 @@ static int verify_buf(int tc, void *buf, off_t off) ptr = buf; for (i = 0; i < u_in_buf; i++) { if (off != *ptr) { - fprintf(stderr, "Found %u, wanted %lu\n", *ptr, off); + fprintf(stderr, "Found %u, wanted %llu\n", *ptr, + (unsigned long long) off); return 1; } ptr++; @@ -206,6 +208,10 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read, goto err; } if (cqe->res != 0) { + if (!no_pt) { + no_pt = 1; + goto skip; + } fprintf(stderr, "cqe res %d, wanted 0\n", cqe->res); goto err; } @@ -235,6 +241,7 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read, } } +skip: close(fd); return 0; err: @@ -259,6 +266,10 @@ static int test_io(const char *file, int tc, int read, int sqthread, if (ret == T_SETUP_SKIP) return 0; if (ret != T_SETUP_OK) { + if (ret == -EINVAL) { + no_pt = 1; + return T_SETUP_SKIP; + } fprintf(stderr, "ring create failed: %d\n", ret); return 1; } @@ -269,8 +280,6 @@ static int test_io(const char *file, int tc, int read, int sqthread, return ret; } -extern unsigned __io_uring_flush_sq(struct io_uring *ring); - /* * Send a passthrough command that nvme will fail during submission. * This comes handy for testing error handling. @@ -283,8 +292,7 @@ static int test_invalid_passthru_submit(const char *file) struct io_uring_sqe *sqe; struct nvme_uring_cmd *cmd; - ring_flags = IORING_SETUP_IOPOLL | IORING_SETUP_SQE128; - ring_flags |= IORING_SETUP_CQE32; + ring_flags = IORING_SETUP_CQE32 | IORING_SETUP_SQE128; ret = t_create_ring(1, &ring, ring_flags); if (ret != T_SETUP_OK) { @@ -347,6 +355,8 @@ static int test_io_uring_submit_enters(const char *file) int fd, i, ret, ring_flags, open_flags; unsigned head; struct io_uring_cqe *cqe; + struct nvme_uring_cmd *cmd; + struct io_uring_sqe *sqe; ring_flags = IORING_SETUP_IOPOLL; ring_flags |= IORING_SETUP_SQE128; @@ -366,12 +376,28 @@ static int test_io_uring_submit_enters(const char *file) } for (i = 0; i < BUFFERS; i++) { - struct io_uring_sqe *sqe; off_t offset = BS * (rand() % BUFFERS); + __u64 slba; + __u32 nlb; sqe = io_uring_get_sqe(&ring); - io_uring_prep_writev(sqe, fd, &vecs[i], 1, offset); - sqe->user_data = 1; + io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset); + sqe->user_data = i; + sqe->opcode = IORING_OP_URING_CMD; + sqe->cmd_op = NVME_URING_CMD_IO; + cmd = (struct nvme_uring_cmd *)sqe->cmd; + memset(cmd, 0, sizeof(struct nvme_uring_cmd)); + + slba = offset >> lba_shift; + nlb = (BS >> lba_shift) - 1; + + cmd->opcode = nvme_cmd_read; + cmd->cdw10 = slba & 0xffffffff; + cmd->cdw11 = slba >> 32; + cmd->cdw12 = nlb; + cmd->addr = (__u64)(uintptr_t)&vecs[i]; + cmd->data_len = 1; + cmd->nsid = nsid; } /* submit manually to avoid adding IORING_ENTER_GETEVENTS */ @@ -427,6 +453,8 @@ int main(int argc, char *argv[]) int nonvec = (i & 8) != 0; ret = test_io(fname, i, read, sqthread, fixed, nonvec); + if (no_pt) + break; if (ret) { fprintf(stderr, "test_io failed %d/%d/%d/%d\n", read, sqthread, fixed, nonvec); @@ -434,6 +462,9 @@ int main(int argc, char *argv[]) } } + if (no_pt) + return T_EXIT_SKIP; + ret = test_io_uring_submit_enters(fname); if (ret) { fprintf(stderr, "test_io_uring_submit_enters failed\n"); diff --git a/contrib/libs/liburing/test/io_uring_register.c b/contrib/libs/liburing/test/io_uring_register.c index b484ef4c4f..1201ffb2dc 100644 --- a/contrib/libs/liburing/test/io_uring_register.c +++ b/contrib/libs/liburing/test/io_uring_register.c @@ -411,7 +411,7 @@ static int ioring_poll(struct io_uring *ring, int fd, int fixed) return 1; } ret = 0; - if (cqe->res != POLLOUT) { + if (!(cqe->res & POLLOUT)) { fprintf(stderr, "io_uring_wait_cqe: expected 0x%.8x, got 0x%.8x\n", POLLOUT, cqe->res); ret = 1; diff --git a/contrib/libs/liburing/test/io_uring_setup.c b/contrib/libs/liburing/test/io_uring_setup.c index 3d5a6c4bca..e0c6bcb519 100644 --- a/contrib/libs/liburing/test/io_uring_setup.c +++ b/contrib/libs/liburing/test/io_uring_setup.c @@ -20,86 +20,10 @@ #include "../syscall.h" -char *features_string(struct io_uring_params *p) -{ - static char flagstr[64]; - - if (!p || !p->features) - return "none"; - - if (p->features & ~IORING_FEAT_SINGLE_MMAP) { - snprintf(flagstr, 64, "0x%.8x", p->features); - return flagstr; - } - - if (p->features & IORING_FEAT_SINGLE_MMAP) - strncat(flagstr, "IORING_FEAT_SINGLE_MMAP", 64 - strlen(flagstr)); - - return flagstr; -} - -/* - * Attempt the call with the given args. Return 0 when expect matches - * the return value of the system call, 1 otherwise. - */ -char * -flags_string(struct io_uring_params *p) -{ - static char flagstr[64]; - int add_pipe = 0; - - memset(flagstr, 0, sizeof(flagstr)); - - if (!p || p->flags == 0) - return "none"; - - /* - * If unsupported flags are present, just print the bitmask. - */ - if (p->flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | - IORING_SETUP_SQ_AFF)) { - snprintf(flagstr, 64, "0x%.8x", p->flags); - return flagstr; - } - - if (p->flags & IORING_SETUP_IOPOLL) { - strncat(flagstr, "IORING_SETUP_IOPOLL", 64 - strlen(flagstr)); - add_pipe = 1; - } - if (p->flags & IORING_SETUP_SQPOLL) { - if (add_pipe) - strncat(flagstr, "|", 64 - strlen(flagstr)); - else - add_pipe = 1; - strncat(flagstr, "IORING_SETUP_SQPOLL", 64 - strlen(flagstr)); - } - if (p->flags & IORING_SETUP_SQ_AFF) { - if (add_pipe) - strncat(flagstr, "|", 64 - strlen(flagstr)); - strncat(flagstr, "IORING_SETUP_SQ_AFF", 64 - strlen(flagstr)); - } - - return flagstr; -} - -char * -dump_resv(struct io_uring_params *p) -{ - static char resvstr[4096]; - - if (!p) - return ""; - - sprintf(resvstr, "0x%.8x 0x%.8x 0x%.8x", p->resv[0], - p->resv[1], p->resv[2]); - - return resvstr; -} - /* bogus: setup returns a valid fd on success... expect can't predict the fd we'll get, so this really only takes 1 parameter: error */ -int -try_io_uring_setup(unsigned entries, struct io_uring_params *p, int expect) +static int try_io_uring_setup(unsigned entries, struct io_uring_params *p, + int expect) { int ret; @@ -124,8 +48,7 @@ try_io_uring_setup(unsigned entries, struct io_uring_params *p, int expect) return 0; } -int -main(int argc, char **argv) +int main(int argc, char **argv) { int fd; unsigned int status = 0; diff --git a/contrib/libs/liburing/test/iopoll-overflow.c b/contrib/libs/liburing/test/iopoll-overflow.c new file mode 100644 index 0000000000..96152fcfe4 --- /dev/null +++ b/contrib/libs/liburing/test/iopoll-overflow.c @@ -0,0 +1,119 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: IOPOLL with overflow test case + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/types.h> +#include <poll.h> +#include <sys/eventfd.h> +#include <sys/resource.h> +#include "helpers.h" +#include "liburing.h" +#include "../src/syscall.h" + +#define FILE_SIZE (128 * 1024) +#define BS 4096 +#define BUFFERS (FILE_SIZE / BS) + +static struct iovec *vecs; + +static int test(struct io_uring *ring, int fd) +{ + struct io_uring_sqe *sqe; + int i, j, ret; + loff_t off; + + off = FILE_SIZE - BS; + for (j = 0; j < 8; j++) { + for (i = 0; i < BUFFERS; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_read(sqe, fd, vecs[i].iov_base, + vecs[i].iov_len, off); + if (!off) + off = FILE_SIZE - BS; + else + off -= BS; + } + ret = io_uring_submit(ring); + if (ret != BUFFERS) { + fprintf(stderr, "submitted %d\n", ret); + return T_EXIT_FAIL; + } + } + + sleep(1); + + ret = __sys_io_uring_enter(ring->ring_fd, 0, BUFFERS * 8, + IORING_ENTER_GETEVENTS, NULL); + + for (i = 0; i < BUFFERS * 8; i++) { + struct io_uring_cqe *cqe; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait=%d\n", ret); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(ring, cqe); + } + + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + struct io_uring_params p = { }; + struct io_uring ring; + char buf[256]; + char *fname; + int ret, fd; + + p.flags = IORING_SETUP_IOPOLL | IORING_SETUP_CQSIZE; + p.cq_entries = 64; + ret = t_create_ring_params(64, &ring, &p); + if (ret == T_SETUP_SKIP) + return 0; + if (ret != T_SETUP_OK) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + if (argc > 1) { + fname = argv[1]; + } else { + srand((unsigned)time(NULL)); + snprintf(buf, sizeof(buf), ".basic-rw-%u-%u", + (unsigned)rand(), (unsigned)getpid()); + fname = buf; + t_create_file(fname, FILE_SIZE); + } + + fd = open(fname, O_RDONLY | O_DIRECT); + if (fd < 0) { + if (errno == EINVAL) { + if (fname != argv[1]) + unlink(fname); + return T_EXIT_SKIP; + } + perror("open"); + goto err; + } + + vecs = t_create_buffers(BUFFERS, BS); + + ret = test(&ring, fd); + + if (fname != argv[1]) + unlink(fname); + return ret; +err: + if (fname != argv[1]) + unlink(fname); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/iopoll-overflow.t/ya.make b/contrib/libs/liburing/test/iopoll-overflow.t/ya.make new file mode 100644 index 0000000000..e8994d965a --- /dev/null +++ b/contrib/libs/liburing/test/iopoll-overflow.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + iopoll-overflow.c +) + +END() diff --git a/contrib/libs/liburing/test/iopoll.c b/contrib/libs/liburing/test/iopoll.c index dfb73265a8..b77b82d945 100644 --- a/contrib/libs/liburing/test/iopoll.c +++ b/contrib/libs/liburing/test/iopoll.c @@ -88,6 +88,8 @@ static int __test_io(const char *file, struct io_uring *ring, int write, int sqt } fd = open(file, open_flags); if (fd < 0) { + if (errno == EINVAL) + return 0; perror("file open"); goto err; } @@ -202,7 +204,80 @@ err: return 1; } -extern unsigned __io_uring_flush_sq(struct io_uring *ring); +static void sig_alrm(int sig) +{ + fprintf(stderr, "Ran out of time for peek test!\n"); + exit(T_EXIT_FAIL); +} + +/* + * if we are polling, io_uring_cqe_peek() always needs to enter the kernel + */ +static int test_io_uring_cqe_peek(const char *file) +{ + struct io_uring_cqe *cqe; + struct io_uring ring; + struct sigaction act; + int fd, i, ret = T_EXIT_FAIL; + + if (no_iopoll) + return 0; + + ret = io_uring_queue_init(64, &ring, IORING_SETUP_IOPOLL); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + fd = open(file, O_RDONLY | O_DIRECT); + if (fd < 0) { + if (errno == EINVAL) { + io_uring_queue_exit(&ring); + return T_EXIT_SKIP; + } + perror("file open"); + goto err; + } + + for (i = 0; i < BUFFERS; i++) { + struct io_uring_sqe *sqe; + off_t offset = BS * (rand() % BUFFERS); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset); + sqe->user_data = 1; + } + + /* + * Set alarm for 5 seconds, we should be done way before that + */ + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_alrm; + sigaction(SIGALRM, &act, NULL); + alarm(5); + + ret = io_uring_submit(&ring); + if (ret != BUFFERS) { + fprintf(stderr, "submit=%d\n", ret); + goto err; + } + + ret = T_EXIT_PASS; + i = 0; + do { + ret = io_uring_peek_cqe(&ring, &cqe); + if (ret) + continue; + io_uring_cqe_seen(&ring, cqe); + i++; + } while (i < BUFFERS); + +err: + if (fd != -1) + close(fd); + io_uring_queue_exit(&ring); + return ret; +} /* * if we are polling io_uring_submit needs to always enter the @@ -228,6 +303,8 @@ static int test_io_uring_submit_enters(const char *file) open_flags = O_WRONLY | O_DIRECT; fd = open(file, open_flags); if (fd < 0) { + if (errno == EINVAL) + return T_EXIT_SKIP; perror("file open"); goto err; } @@ -365,9 +442,18 @@ int main(int argc, char *argv[]) } ret = test_io_uring_submit_enters(fname); - if (ret) { - fprintf(stderr, "test_io_uring_submit_enters failed\n"); - goto err; + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test_io_uring_submit_enters failed\n"); + goto err; + } + + /* + * Keep this last, it exits on failure + */ + ret = test_io_uring_cqe_peek(fname); + if (ret == T_EXIT_FAIL) { + fprintf(stderr, "test_io_uring_cqe_peek failed\n"); + goto err; } if (fname != argv[1]) diff --git a/contrib/libs/liburing/test/lfs-openat-write.c b/contrib/libs/liburing/test/lfs-openat-write.c index 8e3c404de4..c2f14dc308 100644 --- a/contrib/libs/liburing/test/lfs-openat-write.c +++ b/contrib/libs/liburing/test/lfs-openat-write.c @@ -1,10 +1,6 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ -#define _LARGEFILE_SOURCE -#define _FILE_OFFSET_BITS 64 - -#include <liburing.h> #include <string.h> #include <stdio.h> #include <stdlib.h> @@ -15,16 +11,18 @@ #include <sys/resource.h> #include <unistd.h> +#include "liburing.h" #include "helpers.h" static const int RSIZE = 2; -static const int OPEN_FLAGS = O_RDWR | O_CREAT; +static const int OPEN_FLAGS = O_RDWR | O_CREAT | O_LARGEFILE; static const mode_t OPEN_MODE = S_IRUSR | S_IWUSR; -#define DIE(...) do {\ - fprintf(stderr, __VA_ARGS__);\ - abort();\ - } while(0); +#define DIE(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ + abort(); \ + } while(0) static int do_write(struct io_uring *ring, int fd, off_t offset) { diff --git a/contrib/libs/liburing/test/lfs-openat.c b/contrib/libs/liburing/test/lfs-openat.c index 1d93df7e4c..394a7461b4 100644 --- a/contrib/libs/liburing/test/lfs-openat.c +++ b/contrib/libs/liburing/test/lfs-openat.c @@ -1,9 +1,6 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ -#define _LARGEFILE_SOURCE -#define _FILE_OFFSET_BITS 64 - #include <string.h> #include <stdio.h> #include <stdlib.h> @@ -16,13 +13,14 @@ #include "liburing.h" -#define DIE(...) do {\ - fprintf(stderr, __VA_ARGS__);\ - abort();\ - } while(0); +#define DIE(...) \ + do { \ + fprintf(stderr, __VA_ARGS__); \ + abort(); \ + } while(0) static const int RSIZE = 2; -static const int OPEN_FLAGS = O_RDWR | O_CREAT; +static const int OPEN_FLAGS = O_RDWR | O_CREAT | O_LARGEFILE; static const mode_t OPEN_MODE = S_IRUSR | S_IWUSR; static int open_io_uring(struct io_uring *ring, int dfd, const char *fn) diff --git a/contrib/libs/liburing/test/link_drain.c b/contrib/libs/liburing/test/link_drain.c index ec8021f807..3f4e4aef2c 100644 --- a/contrib/libs/liburing/test/link_drain.c +++ b/contrib/libs/liburing/test/link_drain.c @@ -97,7 +97,7 @@ err: return 1; } -int test_link_drain_multi(struct io_uring *ring) +static int test_link_drain_multi(struct io_uring *ring) { struct io_uring_cqe *cqe; struct io_uring_sqe *sqe[9]; @@ -199,15 +199,17 @@ err: } -int main(int argc, char *argv[]) +static int test_drain(bool defer) { struct io_uring ring; int i, ret; + unsigned int flags = 0; - if (argc > 1) - return 0; + if (defer) + flags = IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN; - ret = io_uring_queue_init(100, &ring, 0); + ret = io_uring_queue_init(100, &ring, flags); if (ret) { printf("ring setup failed\n"); return 1; @@ -228,3 +230,27 @@ int main(int argc, char *argv[]) return ret; } + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = test_drain(false); + if (ret) { + fprintf(stderr, "test_drain(false) failed\n"); + return T_EXIT_FAIL; + } + + if (t_probe_defer_taskrun()) { + ret = test_drain(true); + if (ret) { + fprintf(stderr, "test_drain(true) failed\n"); + return T_EXIT_FAIL; + } + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/madvise.c b/contrib/libs/liburing/test/madvise.c index bd44a9741c..20c452c4a6 100644 --- a/contrib/libs/liburing/test/madvise.c +++ b/contrib/libs/liburing/test/madvise.c @@ -183,7 +183,7 @@ int main(int argc, char *argv[]) } /* too hard to reliably test, just ignore */ - if (0 && bad > good) + if ((0) && bad > good) fprintf(stderr, "Suspicious timings (%u > %u)\n", bad, good); if (fname != argv[1]) unlink(fname); diff --git a/contrib/libs/liburing/test/msg-ring-flags.c b/contrib/libs/liburing/test/msg-ring-flags.c new file mode 100644 index 0000000000..d533b8f72d --- /dev/null +++ b/contrib/libs/liburing/test/msg-ring-flags.c @@ -0,0 +1,193 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test ring messaging with flags command + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <pthread.h> + +#include "liburing.h" +#include "helpers.h" + +#define CUSTOM_FLAG 0x42 +#define USER_DATA 0x5aa5 +#define LEN 0x20 +#define ID 0x1 + +struct data { + pthread_barrier_t barrier; + int fd; +}; + +static int recv_msg(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + int ret; + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe %d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->user_data != USER_DATA) { + fprintf(stderr, "user_data %llx\n", (long long) cqe->user_data); + return T_EXIT_FAIL; + } + if (cqe->res != LEN) { + fprintf(stderr, "len %x\n", cqe->res); + return T_EXIT_FAIL; + } + if (cqe->flags != CUSTOM_FLAG) { + fprintf(stderr, "flags %x\n", cqe->flags); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} + +static int send_msg(struct io_uring *ring, int target_fd) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return T_EXIT_FAIL; + } + + io_uring_prep_msg_ring_cqe_flags(sqe, target_fd, LEN, USER_DATA, + 0, CUSTOM_FLAG); + sqe->user_data = ID; + + ret = io_uring_submit(ring); + if (ret <= 0) { + if (ret == -EINVAL) + return T_EXIT_SKIP; + + fprintf(stderr, "sqe submit failed: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return T_EXIT_FAIL; + } + if (cqe->res != 0) { + if (cqe->res == -EINVAL) + return T_EXIT_SKIP; + fprintf(stderr, "cqe res %d\n", cqe->res); + return T_EXIT_FAIL; + } + if (cqe->user_data != ID) { + fprintf(stderr, "user_data %llx\n", (long long) cqe->user_data); + return T_EXIT_FAIL; + } + + io_uring_cqe_seen(ring, cqe); + return T_EXIT_PASS; +} + +static void *thread_fn(void *data) +{ + struct data *d = data; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(2, &ring, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER); + if (ret) { + fprintf(stderr, "ring init failed %d\n", ret); + pthread_barrier_wait(&d->barrier); + return NULL; + } + + d->fd = ring.ring_fd; + pthread_barrier_wait(&d->barrier); + + if (recv_msg(&ring)) + return (void *) 1; + + return NULL; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring, ring2; + pthread_t thread; + struct data d; + void *ret2; + int ret, i; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(2, &ring, 0); + if (ret) { + fprintf(stderr, "io_uring_queue_init failed for ring1: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_queue_init(2, &ring2, 0); + if (ret) { + fprintf(stderr, "io_uring_queue_init failed for ring2: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = send_msg(&ring, ring2.ring_fd); + if (ret) { + if (ret != T_EXIT_SKIP) + fprintf(stderr, "send_msg failed: %d\n", ret); + return ret; + } + + ret = recv_msg(&ring2); + if (ret) { + fprintf(stderr, "recv_msg failed: %d\n", ret); + return ret; + } + + for (i = 0; i < 8; i++) { + ret = send_msg(&ring, ring2.ring_fd); + if (ret) { + if (ret != T_EXIT_SKIP) + fprintf(stderr, "send_msg failed: %d\n", ret); + return ret; + } + } + + for (i = 0; i < 8; i++) { + ret = recv_msg(&ring2); + if (ret) { + fprintf(stderr, "recv_msg failed: %d\n", ret); + return ret; + } + } + + pthread_barrier_init(&d.barrier, NULL, 2); + d.fd = -1; + pthread_create(&thread, NULL, thread_fn, &d); + pthread_barrier_wait(&d.barrier); + if (d.fd == -1) + return T_EXIT_FAIL; + + ret = send_msg(&ring, d.fd); + if (ret) { + fprintf(stderr, "send msg failed: %d\n", ret); + return ret; + } + pthread_join(thread, &ret2); + if (ret2) { + fprintf(stderr, "Remote test failed\n"); + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/msg-ring-flags.t/ya.make b/contrib/libs/liburing/test/msg-ring-flags.t/ya.make new file mode 100644 index 0000000000..1520ca32a0 --- /dev/null +++ b/contrib/libs/liburing/test/msg-ring-flags.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + msg-ring-flags.c +) + +END() diff --git a/contrib/libs/liburing/test/msg-ring-overflow.c b/contrib/libs/liburing/test/msg-ring-overflow.c new file mode 100644 index 0000000000..1a3a6ad200 --- /dev/null +++ b/contrib/libs/liburing/test/msg-ring-overflow.c @@ -0,0 +1,160 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: test ring messaging command + * + */ +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> + +#include "liburing.h" +#include "helpers.h" + +static int no_msg; + +static int test(struct io_uring *ring, unsigned dst_flags) +{ + struct io_uring_params p = { }; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring dst; + int ret, i, err_ret = T_EXIT_FAIL; + + p.flags = dst_flags | IORING_SETUP_CQSIZE; + p.cq_entries = 4; + ret = io_uring_queue_init_params(4, &dst, &p); + if (ret) { + fprintf(stderr, "Destination ring create failed %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < 8; i++) { + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_msg_ring(sqe, dst.ring_fd, 0x10, 0x1234, 0); + sqe->user_data = i + 1; + } + + ret = io_uring_submit(ring); + if (ret != 8) { + /* + * Likely an old kernel that doesn't support the opcode, + * just skip the test. + */ + if (ret == 1) { + err_ret = T_EXIT_SKIP; + no_msg = 1; + goto err; + } + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + for (i = 0; i < 8; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 1 ... 8: + if (cqe->res == -EINVAL || cqe->res == -EOPNOTSUPP) { + no_msg = 1; + goto out; + } + if (cqe->res != 0) { + fprintf(stderr, "cqe res %d\n", cqe->res); + goto err; + } + break; + case 0x1234: + if (cqe->res != 0x10) { + fprintf(stderr, "invalid len %x\n", cqe->res); + goto err; + } + break; + default: + fprintf(stderr, "Invalid user_data\n"); + goto err; + } + io_uring_cqe_seen(ring, cqe); + } + + for (i = 0; i < 8; i++) { + ret = io_uring_wait_cqe(&dst, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + switch (cqe->user_data) { + case 0x1234: + if (cqe->res != 0x10) { + fprintf(stderr, "invalid len %x\n", cqe->res); + goto err; + } + break; + default: + fprintf(stderr, "Invalid user_data\n"); + goto err; + } + io_uring_cqe_seen(&dst, cqe); + } + +out: + io_uring_queue_exit(&dst); + return no_msg ? T_EXIT_SKIP : T_EXIT_PASS; +err: + io_uring_queue_exit(&dst); + return err_ret; +} + +int main(int argc, char *argv[]) +{ + struct io_uring src; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &src, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = test(&src, 0); + if (ret && !no_msg) { + fprintf(stderr, "test failed\n"); + return ret; + } + if (no_msg) + return T_EXIT_SKIP; + + ret = test(&src, IORING_SETUP_IOPOLL); + if (ret) { + fprintf(stderr, "test IOPOLL failed\n"); + return ret; + } + + ret = test(&src, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER); + if (ret) { + fprintf(stderr, "test defer failed\n"); + return ret; + } + + ret = test(&src, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_IOPOLL); + if (ret) { + fprintf(stderr, "test defer IOPOLL failed\n"); + return ret; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/msg-ring-overflow.t/ya.make b/contrib/libs/liburing/test/msg-ring-overflow.t/ya.make new file mode 100644 index 0000000000..ab0076307f --- /dev/null +++ b/contrib/libs/liburing/test/msg-ring-overflow.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + msg-ring-overflow.c +) + +END() diff --git a/contrib/libs/liburing/test/msg-ring.c b/contrib/libs/liburing/test/msg-ring.c index 079c1609d0..221fb7895a 100644 --- a/contrib/libs/liburing/test/msg-ring.c +++ b/contrib/libs/liburing/test/msg-ring.c @@ -94,17 +94,23 @@ static void *wait_cqe_fn(void *data) goto err; } + io_uring_cqe_seen(ring, cqe); return NULL; err: + io_uring_cqe_seen(ring, cqe); return (void *) (unsigned long) 1; } static int test_remote(struct io_uring *ring, struct io_uring *target) { + pthread_t thread; + void *tret; struct io_uring_cqe *cqe; struct io_uring_sqe *sqe; int ret; + pthread_create(&thread, NULL, wait_cqe_fn, target); + sqe = io_uring_get_sqe(ring); if (!sqe) { fprintf(stderr, "get sqe failed\n"); @@ -135,6 +141,80 @@ static int test_remote(struct io_uring *ring, struct io_uring *target) } io_uring_cqe_seen(ring, cqe); + pthread_join(thread, &tret); + return 0; +err: + return 1; +} + +static void *remote_submit_fn(void *data) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring *target = data; + struct io_uring ring; + int ret; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "thread ring setup failed: %d\n", ret); + goto err; + } + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + goto err; + } + + io_uring_prep_msg_ring(sqe, target->ring_fd, 0x20, 0x5aa5, 0); + sqe->user_data = 1; + + ret = io_uring_submit(&ring); + if (ret <= 0) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + goto err; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res != 0 || cqe->user_data != 1) { + fprintf(stderr, "invalid cqe\n"); + goto err; + } + io_uring_cqe_seen(&ring, cqe); + io_uring_queue_exit(&ring); + return NULL; +err: + return (void *) (unsigned long) 1; +} + +static int test_remote_submit(struct io_uring *target) +{ + struct io_uring_cqe *cqe; + pthread_t thread; + void *tret; + int ret; + + pthread_create(&thread, NULL, remote_submit_fn, target); + + ret = io_uring_wait_cqe(target, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + goto err; + } + if (cqe->res != 0x20) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return -1; + } + if (cqe->user_data != 0x5aa5) { + fprintf(stderr, "user_data %llx\n", (long long) cqe->user_data); + return -1; + } + io_uring_cqe_seen(target, cqe); + pthread_join(thread, &tret); return 0; err: return 1; @@ -192,11 +272,52 @@ err: return 1; } +static int test_disabled_ring(struct io_uring *ring, int flags) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring disabled_ring; + int ret; + + flags |= IORING_SETUP_R_DISABLED; + ret = io_uring_queue_init(8, &disabled_ring, flags); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(ring); + io_uring_prep_msg_ring(sqe, disabled_ring.ring_fd, 0x10, 0x1234, 0); + sqe->user_data = 1; + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "wait completion %d\n", ret); + return 1; + } + if (cqe->res != 0 && cqe->res != -EBADFD) { + fprintf(stderr, "cqe res %d\n", cqe->res); + return 1; + } + if (cqe->user_data != 1) { + fprintf(stderr, "user_data %llx\n", (long long) cqe->user_data); + return 1; + } + + io_uring_cqe_seen(ring, cqe); + io_uring_queue_exit(&disabled_ring); + return 0; +} + int main(int argc, char *argv[]) { struct io_uring ring, ring2, pring; - pthread_t thread; - void *tret; int ret, i; if (argc > 1) @@ -221,41 +342,80 @@ int main(int argc, char *argv[]) ret = test_own(&ring); if (ret) { fprintf(stderr, "test_own failed\n"); - return ret; + return T_EXIT_FAIL; } - if (no_msg) { - fprintf(stdout, "Skipped\n"); + if (no_msg) return T_EXIT_SKIP; - } ret = test_own(&pring); if (ret) { fprintf(stderr, "test_own iopoll failed\n"); - return ret; + return T_EXIT_FAIL; } ret = test_invalid(&ring, 0); if (ret) { fprintf(stderr, "test_invalid failed\n"); - return ret; + return T_EXIT_FAIL; } for (i = 0; i < 2; i++) { ret = test_invalid(&ring, 1); if (ret) { fprintf(stderr, "test_invalid fixed failed\n"); - return ret; + return T_EXIT_FAIL; } } - pthread_create(&thread, NULL, wait_cqe_fn, &ring2); - ret = test_remote(&ring, &ring2); if (ret) { fprintf(stderr, "test_remote failed\n"); - return ret; + return T_EXIT_FAIL; } - pthread_join(thread, &tret); + io_uring_queue_exit(&ring); + io_uring_queue_exit(&pring); + + if (t_probe_defer_taskrun()) { + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN); + if (ret) { + fprintf(stderr, "deferred ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + ret = test_own(&ring); + if (ret) { + fprintf(stderr, "test_own deferred failed\n"); + return T_EXIT_FAIL; + } + + for (i = 0; i < 2; i++) { + ret = test_invalid(&ring, i); + if (ret) { + fprintf(stderr, "test_invalid(0) deferred failed\n"); + return T_EXIT_FAIL; + } + } + + ret = test_remote_submit(&ring); + if (ret) { + fprintf(stderr, "test_remote_submit failed\n"); + return T_EXIT_FAIL; + } + io_uring_queue_exit(&ring); + + if (test_disabled_ring(&ring2, 0)) { + fprintf(stderr, "test_disabled_ring failed\n"); + return T_EXIT_FAIL; + } + + if (test_disabled_ring(&ring2, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN)) { + fprintf(stderr, "test_disabled_ring defer failed\n"); + return T_EXIT_FAIL; + } + } + io_uring_queue_exit(&ring2); return T_EXIT_PASS; } diff --git a/contrib/libs/liburing/test/multicqes_drain.c b/contrib/libs/liburing/test/multicqes_drain.c index 99e5fe1247..fb83441219 100644 --- a/contrib/libs/liburing/test/multicqes_drain.c +++ b/contrib/libs/liburing/test/multicqes_drain.c @@ -43,12 +43,16 @@ struct sqe_info { * up an entry in multi_sqes when form a cancellation sqe. * multi_cap: limitation of number of multishot sqes */ -const unsigned sqe_flags[4] = {0, IOSQE_IO_LINK, IOSQE_IO_DRAIN, - IOSQE_IO_LINK | IOSQE_IO_DRAIN}; -int multi_sqes[max_entry], cnt = 0; -int multi_cap = max_entry / 5; +static const unsigned sqe_flags[4] = { + 0, + IOSQE_IO_LINK, + IOSQE_IO_DRAIN, + IOSQE_IO_LINK | IOSQE_IO_DRAIN +}; +static int multi_sqes[max_entry], cnt = 0; +static int multi_cap = max_entry / 5; -int write_pipe(int pipe, char *str) +static int write_pipe(int pipe, char *str) { int ret; do { @@ -58,7 +62,7 @@ int write_pipe(int pipe, char *str) return ret; } -void read_pipe(int pipe) +static void read_pipe(int pipe) { char str[4] = {0}; int ret; @@ -68,18 +72,21 @@ void read_pipe(int pipe) perror("read"); } -int trigger_event(int p[]) +static int trigger_event(struct io_uring *ring, int p[]) { int ret; if ((ret = write_pipe(p[1], "foo")) != 3) { fprintf(stderr, "bad write return %d\n", ret); return 1; } + usleep(1000); + io_uring_get_events(ring); read_pipe(p[0]); return 0; } -void io_uring_sqe_prep(int op, struct io_uring_sqe *sqe, unsigned sqe_flags, int arg) +static void io_uring_sqe_prep(int op, struct io_uring_sqe *sqe, + unsigned sqe_flags, int arg) { switch (op) { case multi: @@ -99,7 +106,7 @@ void io_uring_sqe_prep(int op, struct io_uring_sqe *sqe, unsigned sqe_flags, int sqe->flags = sqe_flags; } -__u8 generate_flags(int sqe_op) +static __u8 generate_flags(int sqe_op) { __u8 flags = 0; /* @@ -137,7 +144,7 @@ __u8 generate_flags(int sqe_op) * - ensure number of multishot sqes doesn't exceed multi_cap * - don't generate multishot sqes after high watermark */ -int generate_opcode(int i, int pre_flags) +static int generate_opcode(int i, int pre_flags) { int sqe_op; int high_watermark = max_entry - max_entry / 5; @@ -164,7 +171,7 @@ static inline void add_multishot_sqe(int index) multi_sqes[cnt++] = index; } -int remove_multishot_sqe() +static int remove_multishot_sqe(void) { int ret; @@ -232,10 +239,8 @@ static int test_generic_drain(struct io_uring *ring) if (si[i].op != multi && si[i].op != single) continue; - if (trigger_event(pipes[i])) + if (trigger_event(ring, pipes[i])) goto err; - - io_uring_get_events(ring); } sleep(1); i = 0; @@ -313,13 +318,11 @@ static int test_simple_drain(struct io_uring *ring) } for (i = 0; i < 2; i++) { - if (trigger_event(pipe1)) + if (trigger_event(ring, pipe1)) goto err; - io_uring_get_events(ring); } - if (trigger_event(pipe2)) - goto err; - io_uring_get_events(ring); + if (trigger_event(ring, pipe2)) + goto err; for (i = 0; i < 2; i++) { sqe[i] = io_uring_get_sqe(ring); diff --git a/contrib/libs/liburing/test/nvme.h b/contrib/libs/liburing/test/nvme.h index 53ad1142a8..87abf8a64b 100644 --- a/contrib/libs/liburing/test/nvme.h +++ b/contrib/libs/liburing/test/nvme.h @@ -57,8 +57,8 @@ enum nvme_io_opcode { nvme_cmd_read = 0x02, }; -int nsid; -__u32 lba_shift; +static int nsid; +static __u32 lba_shift; struct nvme_lbaf { __le16 ms; @@ -120,7 +120,8 @@ static inline int ilog2(uint32_t i) return log; } -int nvme_get_info(const char *file) +__attribute__((__unused__)) +static int nvme_get_info(const char *file) { struct nvme_id_ns ns; int fd, err; diff --git a/contrib/libs/liburing/test/pipe-bug.c b/contrib/libs/liburing/test/pipe-bug.c new file mode 100644 index 0000000000..cb45f6fa1d --- /dev/null +++ b/contrib/libs/liburing/test/pipe-bug.c @@ -0,0 +1,96 @@ +#include "../config-host.h" +// SPDX-License-Identifier: MIT + +/* + * Description: tests bug fixed in + * "io_uring: don't gate task_work run on TIF_NOTIFY_SIGNAL" + * + * See: https://github.com/axboe/liburing/issues/665 + */ + +#include <stdio.h> +#include <string.h> +#include <unistd.h> + +#include "helpers.h" +#include "liburing.h" + +#define CHECK(x) \ +do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d %s failed\n", __FILE__, __LINE__, #x); \ + return -1; \ + } \ +} while (0) + +static int pipe_bug(void) +{ + struct io_uring_params p; + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + char buf[1024]; + int fds[2]; + struct __kernel_timespec to = { + .tv_sec = 1 + }; + + CHECK(pipe(fds) == 0); + + memset(&p, 0, sizeof(p)); + CHECK(t_create_ring_params(8, &ring, &p) == 0); + + /* WRITE */ + sqe = io_uring_get_sqe(&ring); + CHECK(sqe); + io_uring_prep_write(sqe, fds[1], "foobar", strlen("foobar"), 0); /* or -1 */ + CHECK(io_uring_submit(&ring) == 1); + CHECK(io_uring_wait_cqe(&ring, &cqe) == 0); + + io_uring_cqe_seen(&ring, cqe); + + /* CLOSE */ + sqe = io_uring_get_sqe(&ring); + CHECK(sqe); + io_uring_prep_close(sqe, fds[1]); + CHECK(io_uring_submit(&ring) == 1); + CHECK(io_uring_wait_cqe_timeout(&ring, &cqe, &to) == 0); + io_uring_cqe_seen(&ring, cqe); + + /* READ */ + sqe = io_uring_get_sqe(&ring); + CHECK(sqe); + io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0); /* or -1 */ + CHECK(io_uring_submit(&ring) == 1); + CHECK(io_uring_wait_cqe_timeout(&ring, &cqe, &to) == 0); + io_uring_cqe_seen(&ring, cqe); + memset(buf, 0, sizeof(buf)); + + /* READ */ + sqe = io_uring_get_sqe(&ring); + CHECK(sqe); + io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0); /* or -1 */ + CHECK(io_uring_submit(&ring) == 1); + CHECK(io_uring_wait_cqe_timeout(&ring, &cqe, &to) == 0); + io_uring_cqe_seen(&ring, cqe); + + close(fds[0]); + io_uring_queue_exit(&ring); + + return 0; +} + +int main(int argc, char *argv[]) +{ + int i; + + if (argc > 1) + return T_EXIT_SKIP; + + for (i = 0; i < 10000; i++) { + if (pipe_bug()) + return T_EXIT_FAIL; + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/pipe-bug.t/ya.make b/contrib/libs/liburing/test/pipe-bug.t/ya.make new file mode 100644 index 0000000000..1304228eb8 --- /dev/null +++ b/contrib/libs/liburing/test/pipe-bug.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + pipe-bug.c +) + +END() diff --git a/contrib/libs/liburing/test/poll-link.c b/contrib/libs/liburing/test/poll-link.c index 27346c65ae..fda1182e7a 100644 --- a/contrib/libs/liburing/test/poll-link.c +++ b/contrib/libs/liburing/test/poll-link.c @@ -17,8 +17,8 @@ #include "helpers.h" #include "liburing.h" -pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; static int recv_thread_ready = 0; static int recv_thread_done = 0; @@ -72,7 +72,7 @@ static void *send_thread(void *arg) return 0; } -void *recv_thread(void *arg) +static void *recv_thread(void *arg) { struct sockaddr_in addr = { }; struct data *data = arg; diff --git a/contrib/libs/liburing/test/poll-many.c b/contrib/libs/liburing/test/poll-many.c index 8f6a89efdd..fb00cf6327 100644 --- a/contrib/libs/liburing/test/poll-many.c +++ b/contrib/libs/liburing/test/poll-many.c @@ -15,6 +15,7 @@ #include <fcntl.h> #include "liburing.h" +#include "helpers.h" #define NFILES 5000 #define BATCH 500 @@ -138,6 +139,21 @@ static int arm_polls(struct io_uring *ring) return 0; } +static int do_test(struct io_uring *ring) +{ + int i; + + if (arm_polls(ring)) + return 1; + + for (i = 0; i < NLOOPS; i++) { + trigger_polls(); + if (reap_polls(ring)) + return 1; + } + return 0; +} + int main(int argc, char *argv[]) { struct io_uring ring; @@ -150,7 +166,7 @@ int main(int argc, char *argv[]) if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) { perror("getrlimit"); - goto err_noring; + return T_EXIT_FAIL; } if (rlim.rlim_cur < (2 * NFILES + 5)) { @@ -160,14 +176,14 @@ int main(int argc, char *argv[]) if (errno == EPERM) goto err_nofail; perror("setrlimit"); - goto err_noring; + return T_EXIT_FAIL; } } for (i = 0; i < NFILES; i++) { if (pipe(p[i].fd) < 0) { perror("pipe"); - goto err_noring; + return T_EXIT_FAIL; } } @@ -177,31 +193,37 @@ int main(int argc, char *argv[]) if (ret) { if (ret == -EINVAL) { fprintf(stdout, "No CQSIZE, trying without\n"); - ret = io_uring_queue_init(RING_SIZE, &ring, 0); + + params.flags &= ~IORING_SETUP_CQSIZE; + params.cq_entries = 0; + ret = io_uring_queue_init_params(RING_SIZE, &ring, ¶ms); if (ret) { fprintf(stderr, "ring setup failed: %d\n", ret); - return 1; + return T_EXIT_FAIL; } } } - if (arm_polls(&ring)) - goto err; - - for (i = 0; i < NLOOPS; i++) { - trigger_polls(); - ret = reap_polls(&ring); - if (ret) - goto err; + if (do_test(&ring)) { + fprintf(stderr, "test (normal) failed\n"); + return T_EXIT_FAIL; } - io_uring_queue_exit(&ring); + + if (t_probe_defer_taskrun()) { + params.flags |= IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN; + ret = io_uring_queue_init_params(RING_SIZE, &ring, ¶ms); + if (ret) { + fprintf(stderr, "ring DEFER setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + if (do_test(&ring)) { + fprintf(stderr, "test (DEFER) failed\n"); + return T_EXIT_FAIL; + } + io_uring_queue_exit(&ring); + } return 0; -err: - io_uring_queue_exit(&ring); -err_noring: - fprintf(stderr, "poll-many failed\n"); - return 1; err_nofail: fprintf(stderr, "poll-many: not enough files available (and not root), " "skipped\n"); diff --git a/contrib/libs/liburing/test/poll-mshot-overflow.c b/contrib/libs/liburing/test/poll-mshot-overflow.c index b31633d507..94a82bd64d 100644 --- a/contrib/libs/liburing/test/poll-mshot-overflow.c +++ b/contrib/libs/liburing/test/poll-mshot-overflow.c @@ -13,7 +13,7 @@ #include "liburing.h" #include "helpers.h" -int check_final_cqe(struct io_uring *ring) +static int check_final_cqe(struct io_uring *ring) { struct io_uring_cqe *cqe; int count = 0; @@ -138,20 +138,123 @@ static int test(bool defer_taskrun) return ret; } +static int test_downgrade(bool support_defer) +{ + struct io_uring_cqe cqes[128]; + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + int fds[2]; + int ret, i, cqe_count, tmp = 0, more_cqe_count; + + if (pipe(fds) != 0) { + perror("pipe"); + return -1; + } + + struct io_uring_params params = { + .flags = IORING_SETUP_CQSIZE, + .cq_entries = 2 + }; + + ret = io_uring_queue_init_params(2, &ring, ¶ms); + if (ret) { + fprintf(stderr, "queue init: %d\n", ret); + return -1; + } + + sqe = io_uring_get_sqe(&ring); + if (!sqe) { + fprintf(stderr, "get sqe failed\n"); + return -1; + } + io_uring_prep_poll_multishot(sqe, fds[0], POLLIN); + io_uring_sqe_set_data64(sqe, 1); + io_uring_submit(&ring); + + for (i = 0; i < 8; i++) { + ret = write(fds[1], &tmp, sizeof(tmp)); + if (ret != sizeof(tmp)) { + perror("write"); + return -1; + } + ret = read(fds[0], &tmp, sizeof(tmp)); + if (ret != sizeof(tmp)) { + perror("read"); + return -1; + } + } + + cqe_count = 0; + while (!io_uring_peek_cqe(&ring, &cqe)) { + cqes[cqe_count++] = *cqe; + io_uring_cqe_seen(&ring, cqe); + } + + /* Some kernels might allow overflows to poll, + * but if they didn't it should stop the MORE flag + */ + if (cqe_count < 3) { + fprintf(stderr, "too few cqes: %d\n", cqe_count); + return -1; + } else if (cqe_count == 8) { + more_cqe_count = cqe_count; + /* downgrade only available since support_defer */ + if (support_defer) { + fprintf(stderr, "did not downgrade on overflow\n"); + return -1; + } + } else { + more_cqe_count = cqe_count - 1; + cqe = &cqes[cqe_count - 1]; + if (cqe->flags & IORING_CQE_F_MORE) { + fprintf(stderr, "incorrect MORE flag %x\n", cqe->flags); + return -1; + } + } + + for (i = 0; i < more_cqe_count; i++) { + cqe = &cqes[i]; + if (!(cqe->flags & IORING_CQE_F_MORE)) { + fprintf(stderr, "missing MORE flag\n"); + return -1; + } + if (cqe->res < 0) { + fprintf(stderr, "bad res: %d\n", cqe->res); + return -1; + } + } + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + int main(int argc, char *argv[]) { int ret; + bool support_defer; if (argc > 1) return T_EXIT_SKIP; + support_defer = t_probe_defer_taskrun(); + ret = test_downgrade(support_defer); + if (ret) { + fprintf(stderr, "%s: test_downgrade(%d) failed\n", argv[0], support_defer); + return T_EXIT_FAIL; + } + ret = test(false); + if (ret == T_EXIT_SKIP) + return ret; if (ret != T_EXIT_PASS) { fprintf(stderr, "%s: test(false) failed\n", argv[0]); return ret; } - if (t_probe_defer_taskrun()) { + if (support_defer) { ret = test(true); if (ret != T_EXIT_PASS) { fprintf(stderr, "%s: test(true) failed\n", argv[0]); diff --git a/contrib/libs/liburing/test/poll-race-mshot.c b/contrib/libs/liburing/test/poll-race-mshot.c new file mode 100644 index 0000000000..5b2755fab4 --- /dev/null +++ b/contrib/libs/liburing/test/poll-race-mshot.c @@ -0,0 +1,277 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: check that racing wakeups don't re-issue a poll multishot, + * this can leak ring provided buffers. also test if ring + * provided buffers for regular receive can leak if we hit a + * poll race. + */ +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <pthread.h> +#include <string.h> +#include <sys/socket.h> + +#include "liburing.h" +#include "helpers.h" + +#define NREQS 64 +#define BUF_SIZE 64 + +static int no_buf_ring; + +struct data { + pthread_barrier_t barrier; + int fd; +}; + +static void *thread(void *data) +{ + struct data *d = data; + char buf[BUF_SIZE]; + int ret, i, fd; + + memset(buf, 0x5a, BUF_SIZE); + pthread_barrier_wait(&d->barrier); + fd = d->fd; + for (i = 0; i < NREQS; i++) { + ret = write(fd, buf, sizeof(buf)); + if (ret != BUF_SIZE) { + if (ret < 0) { + perror("write"); + printf("bad fd %d\n", fd); + } else + fprintf(stderr, "wrote short %d\n", ret); + } + } + return NULL; +} + +static int test(struct io_uring *ring, struct data *d) +{ + struct io_uring_buf_ring *br; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int fd[2], ret, i; + pthread_t t; + void *buf, *ptr; + void *ret2; + + if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fd) < 0) { + perror("socketpair"); + return T_EXIT_FAIL; + } + + d->fd = fd[1]; + + if (posix_memalign((void **) &buf, 16384, BUF_SIZE * NREQS)) + return T_EXIT_FAIL; + + br = io_uring_setup_buf_ring(ring, NREQS, 1, 0, &ret); + if (!br) { + if (ret == -EINVAL) { + no_buf_ring = 1; + return T_EXIT_SKIP; + } + fprintf(stderr, "buf ring reg %d\n", ret); + return T_EXIT_FAIL; + } + + ptr = buf; + for (i = 0; i < NREQS; i++) { + io_uring_buf_ring_add(br, ptr, BUF_SIZE, i + 1, + io_uring_buf_ring_mask(NREQS), i); + ptr += BUF_SIZE; + } + io_uring_buf_ring_advance(br, NREQS); + + pthread_create(&t, NULL, thread, d); + + for (i = 0; i < NREQS; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_recv(sqe, fd[0], NULL, 0, 0); + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = 1; + } + + pthread_barrier_wait(&d->barrier); + + ret = io_uring_submit(ring); + if (ret != NREQS) { + fprintf(stderr, "submit %d\n", ret); + return T_EXIT_FAIL; + } + + i = 0; + do { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "cqe wait %d\n", ret); + return T_EXIT_FAIL; + } + i++; + if (cqe->res != BUF_SIZE) { + fprintf(stderr, "Bad cqe res %d\n", cqe->res); + break; + } + if (cqe->flags & IORING_CQE_F_BUFFER) { + int bid = cqe->flags >> 16; + + if (bid > NREQS) { + fprintf(stderr, "Bad BID %d\n", bid); + return T_EXIT_FAIL; + } + } else { + fprintf(stderr, "No BID set!\n"); + printf("ret=%d\n", cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(ring, cqe); + if (i > NREQS) { + fprintf(stderr, "Got too many requests?\n"); + return T_EXIT_FAIL; + } + } while (i < NREQS); + + pthread_join(t, &ret2); + free(buf); + io_uring_free_buf_ring(ring, br, NREQS, 1); + close(fd[0]); + close(fd[1]); + return T_EXIT_PASS; +} + +static int test_mshot(struct io_uring *ring, struct data *d) +{ + struct io_uring_buf_ring *br; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int fd[2], ret, i; + pthread_t t; + void *buf, *ptr; + void *ret2; + + if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fd) < 0) { + perror("socketpair"); + return T_EXIT_FAIL; + } + + d->fd = fd[1]; + + if (posix_memalign((void *) &buf, 16384, BUF_SIZE * NREQS)) + return T_EXIT_FAIL; + + br = io_uring_setup_buf_ring(ring, NREQS, 1, 0, &ret); + if (!br) { + fprintf(stderr, "buf ring reg %d\n", ret); + return T_EXIT_FAIL; + } + + ptr = buf; + for (i = 0; i < NREQS; i++) { + io_uring_buf_ring_add(br, ptr, BUF_SIZE, i + 1, + io_uring_buf_ring_mask(NREQS), i); + ptr += BUF_SIZE; + } + io_uring_buf_ring_advance(br, NREQS); + + pthread_create(&t, NULL, thread, d); + + sqe = io_uring_get_sqe(ring); + io_uring_prep_recv_multishot(sqe, fd[0], NULL, 0, 0); + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = 1; + + pthread_barrier_wait(&d->barrier); + + ret = io_uring_submit(ring); + if (ret != 1) { + fprintf(stderr, "submit %d\n", ret); + return T_EXIT_FAIL; + } + + i = 0; + do { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "cqe wait %d\n", ret); + return T_EXIT_FAIL; + } + i++; + if (!(cqe->flags & IORING_CQE_F_MORE)) + break; + if (cqe->res != BUF_SIZE) { + fprintf(stderr, "Bad cqe res %d\n", cqe->res); + break; + } + if (cqe->flags & IORING_CQE_F_BUFFER) { + int bid = cqe->flags >> 16; + + if (bid > NREQS) { + fprintf(stderr, "Bad BID %d\n", bid); + return T_EXIT_FAIL; + } + } else { + fprintf(stderr, "No BID set!\n"); + printf("ret=%d\n", cqe->res); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(ring, cqe); + if (i > NREQS) { + fprintf(stderr, "Got too many requests?\n"); + return T_EXIT_FAIL; + } + } while (1); + + if (i != NREQS + 1) { + fprintf(stderr, "Only got %d requests\n", i); + return T_EXIT_FAIL; + } + + pthread_join(t, &ret2); + io_uring_free_buf_ring(ring, br, NREQS, 1); + free(buf); + close(fd[0]); + close(fd[1]); + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct data d; + int i, ret; + + if (argc > 1) + return T_EXIT_SKIP; + + pthread_barrier_init(&d.barrier, NULL, 2); + + for (i = 0; i < 1000; i++) { + io_uring_queue_init(NREQS, &ring, 0); + ret = test(&ring, &d); + if (ret != T_EXIT_PASS) { + if (no_buf_ring) + break; + fprintf(stderr, "Test failed loop %d\n", i); + return T_EXIT_FAIL; + } + io_uring_queue_exit(&ring); + } + + if (no_buf_ring) + return T_EXIT_SKIP; + + for (i = 0; i < 1000; i++) { + io_uring_queue_init(NREQS, &ring, 0); + ret = test_mshot(&ring, &d); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "Test mshot failed loop %d\n", i); + return T_EXIT_FAIL; + } + io_uring_queue_exit(&ring); + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/poll-race-mshot.t/ya.make b/contrib/libs/liburing/test/poll-race-mshot.t/ya.make new file mode 100644 index 0000000000..14ebf80acd --- /dev/null +++ b/contrib/libs/liburing/test/poll-race-mshot.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + poll-race-mshot.c +) + +END() diff --git a/contrib/libs/liburing/test/poll-race.c b/contrib/libs/liburing/test/poll-race.c new file mode 100644 index 0000000000..37e1fb92d9 --- /dev/null +++ b/contrib/libs/liburing/test/poll-race.c @@ -0,0 +1,106 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: check that multiple receives on the same socket don't get + * stalled if multiple wakers race with the socket readiness. + */ +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <pthread.h> +#include <sys/socket.h> + +#include "liburing.h" +#include "helpers.h" + +#define NREQS 64 + +struct data { + pthread_barrier_t barrier; + int fd; +}; + +static void *thread(void *data) +{ + struct data *d = data; + char buf[64]; + int ret, i; + + pthread_barrier_wait(&d->barrier); + for (i = 0; i < NREQS; i++) { + ret = write(d->fd, buf, sizeof(buf)); + if (ret != 64) + fprintf(stderr, "wrote short %d\n", ret); + } + return NULL; +} + +static int test(struct io_uring *ring, struct data *d) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + int fd[2], ret, i; + char buf[64]; + pthread_t t; + void *ret2; + + if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fd) < 0) { + perror("socketpair"); + return T_EXIT_FAIL; + } + + d->fd = fd[1]; + + pthread_create(&t, NULL, thread, d); + + for (i = 0; i < NREQS; i++) { + sqe = io_uring_get_sqe(ring); + io_uring_prep_recv(sqe, fd[0], buf, sizeof(buf), 0); + } + + pthread_barrier_wait(&d->barrier); + + ret = io_uring_submit(ring); + if (ret != NREQS) { + fprintf(stderr, "submit %d\n", ret); + return T_EXIT_FAIL; + } + + for (i = 0; i < NREQS; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "cqe wait %d\n", ret); + return T_EXIT_FAIL; + } + io_uring_cqe_seen(ring, cqe); + } + + close(fd[0]); + close(fd[1]); + pthread_join(t, &ret2); + return T_EXIT_PASS; +} + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + struct data d; + int i, ret; + + if (argc > 1) + return T_EXIT_SKIP; + + pthread_barrier_init(&d.barrier, NULL, 2); + + io_uring_queue_init(NREQS, &ring, 0); + + for (i = 0; i < 1000; i++) { + ret = test(&ring, &d); + if (ret != T_EXIT_PASS) { + fprintf(stderr, "Test failed\n"); + return T_EXIT_FAIL; + } + } + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/poll-race.t/ya.make b/contrib/libs/liburing/test/poll-race.t/ya.make new file mode 100644 index 0000000000..2333a17060 --- /dev/null +++ b/contrib/libs/liburing/test/poll-race.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + poll-race.c +) + +END() diff --git a/contrib/libs/liburing/test/poll.c b/contrib/libs/liburing/test/poll.c index cfba077af6..7c54e6239f 100644 --- a/contrib/libs/liburing/test/poll.c +++ b/contrib/libs/liburing/test/poll.c @@ -12,16 +12,27 @@ #include <signal.h> #include <poll.h> #include <sys/wait.h> +#include <assert.h> +#include "helpers.h" #include "liburing.h" -static void sig_alrm(int sig) +static void do_setsockopt(int fd, int level, int optname, int val) { - fprintf(stderr, "Timed out!\n"); - exit(1); + if (setsockopt(fd, level, optname, &val, sizeof(val))) + t_error(1, errno, "setsockopt %d.%d: %d", level, optname, val); } -int main(int argc, char *argv[]) +static bool check_cq_empty(struct io_uring *ring) +{ + struct io_uring_cqe *cqe = NULL; + int ret; + + ret = io_uring_peek_cqe(ring, &cqe); /* nothing should be there */ + return ret == -EAGAIN; +} + +static int test_basic(void) { struct io_uring_cqe *cqe; struct io_uring_sqe *sqe; @@ -30,34 +41,22 @@ int main(int argc, char *argv[]) pid_t p; int ret; - if (argc > 1) - return 0; - if (pipe(pipe1) != 0) { perror("pipe"); return 1; } p = fork(); - switch (p) { - case -1: + if (p == -1) { perror("fork"); exit(2); - case 0: { - struct sigaction act; - + } else if (p == 0) { ret = io_uring_queue_init(1, &ring, 0); if (ret) { fprintf(stderr, "child: ring setup failed: %d\n", ret); return 1; } - memset(&act, 0, sizeof(act)); - act.sa_handler = sig_alrm; - act.sa_flags = SA_RESTART; - sigaction(SIGALRM, &act, NULL); - alarm(1); - sqe = io_uring_get_sqe(&ring); if (!sqe) { fprintf(stderr, "get sqe failed\n"); @@ -93,18 +92,237 @@ int main(int argc, char *argv[]) (long) cqe->res); return 1; } + + io_uring_queue_exit(&ring); exit(0); - } - default: - do { - errno = 0; - ret = write(pipe1[1], "foo", 3); - } while (ret == -1 && errno == EINTR); + } + + do { + errno = 0; + ret = write(pipe1[1], "foo", 3); + } while (ret == -1 && errno == EINTR); + + if (ret != 3) { + fprintf(stderr, "parent: bad write return %d\n", ret); + return 1; + } + close(pipe1[0]); + close(pipe1[1]); + return 0; +} + +static int test_missing_events(void) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + int i, ret, sp[2]; + char buf[2] = {}; + int res_mask = 0; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } - if (ret != 3) { - fprintf(stderr, "parent: bad write return %d\n", ret); + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0) { + perror("Failed to create Unix-domain socket pair\n"); + return 1; + } + do_setsockopt(sp[0], SOL_SOCKET, SO_SNDBUF, 1); + ret = send(sp[0], buf, sizeof(buf), 0); + if (ret != sizeof(buf)) { + perror("send failed\n"); + return 1; + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_multishot(sqe, sp[0], POLLIN|POLLOUT); + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "sqe submit failed: %d\n", ret); + return 1; + } + + /* trigger EPOLLIN */ + ret = send(sp[1], buf, sizeof(buf), 0); + if (ret != sizeof(buf)) { + fprintf(stderr, "send sp[1] failed %i %i\n", ret, errno); + return 1; + } + + /* trigger EPOLLOUT */ + ret = recv(sp[1], buf, sizeof(buf), 0); + if (ret != sizeof(buf)) { + perror("recv failed\n"); + return 1; + } + + for (i = 0; ; i++) { + if (i == 0) + ret = io_uring_wait_cqe(&ring, &cqe); + else + ret = io_uring_peek_cqe(&ring, &cqe); + + if (i != 0 && ret == -EAGAIN) { + break; + } + if (ret) { + fprintf(stderr, "wait completion %d, %i\n", ret, i); return 1; } + res_mask |= cqe->res; + io_uring_cqe_seen(&ring, cqe); + } + + if ((res_mask & (POLLIN|POLLOUT)) != (POLLIN|POLLOUT)) { + fprintf(stderr, "missing poll events %i\n", res_mask); + return 1; + } + io_uring_queue_exit(&ring); + close(sp[0]); + close(sp[1]); + return 0; +} + +#define NR_SQES 2048 + +static int test_self_poll(void) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring; + int ret, i, j; + + ret = io_uring_queue_init(NR_SQES, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return T_EXIT_FAIL; + } + + for (j = 0; j < 32; j++) { + for (i = 0; i < NR_SQES; i++) { + sqe = io_uring_get_sqe(&ring); + io_uring_prep_poll_add(sqe, ring.ring_fd, POLLIN); + } + + ret = io_uring_submit(&ring); + assert(ret == NR_SQES); + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + ret = io_uring_submit(&ring); + assert(ret == 1); + + ret = io_uring_wait_cqe(&ring, &cqe); + io_uring_cqe_seen(&ring, cqe); + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; +} + +static int test_disabled_ring_lazy_polling(int early_poll) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct io_uring ring, ring2; + unsigned head; + int ret, i = 0; + + ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | + IORING_SETUP_DEFER_TASKRUN | + IORING_SETUP_R_DISABLED); + if (ret) { + fprintf(stderr, "ring setup failed: %d\n", ret); + return 1; + } + ret = io_uring_queue_init(8, &ring2, 0); + if (ret) { + fprintf(stderr, "ring2 setup failed: %d\n", ret); + return 1; + } + + if (early_poll) { + /* start polling disabled DEFER_TASKRUN ring */ + sqe = io_uring_get_sqe(&ring2); + io_uring_prep_poll_add(sqe, ring.ring_fd, POLLIN); + ret = io_uring_submit(&ring2); + assert(ret == 1); + assert(check_cq_empty(&ring2)); + } + + /* enable rings, which should also activate pollwq */ + ret = io_uring_enable_rings(&ring); + assert(ret >= 0); + + if (!early_poll) { + /* start polling enabled DEFER_TASKRUN ring */ + sqe = io_uring_get_sqe(&ring2); + io_uring_prep_poll_add(sqe, ring.ring_fd, POLLIN); + ret = io_uring_submit(&ring2); + assert(ret == 1); + assert(check_cq_empty(&ring2)); + } + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_nop(sqe); + ret = io_uring_submit(&ring); + assert(ret == 1); + + io_uring_for_each_cqe(&ring2, head, cqe) { + i++; + } + if (i != 1) { + fprintf(stderr, "fail, polling stuck\n"); + return 1; + } + io_uring_queue_exit(&ring); + io_uring_queue_exit(&ring2); + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret; + + if (argc > 1) return 0; + + ret = test_basic(); + if (ret) { + fprintf(stderr, "test_basic() failed %i\n", ret); + return T_EXIT_FAIL; } + + + if (t_probe_defer_taskrun()) { + ret = test_missing_events(); + if (ret) { + fprintf(stderr, "test_missing_events() failed %i\n", ret); + return T_EXIT_FAIL; + } + + ret = test_disabled_ring_lazy_polling(false); + if (ret) { + fprintf(stderr, "test_disabled_ring_lazy_polling(false) failed %i\n", ret); + return T_EXIT_FAIL; + } + + ret = test_disabled_ring_lazy_polling(true); + if (ret) { + fprintf(stderr, "test_disabled_ring_lazy_polling(true) failed %i\n", ret); + return T_EXIT_FAIL; + } + } + + ret = test_self_poll(); + if (ret) { + fprintf(stderr, "test_self_poll failed\n"); + return T_EXIT_FAIL; + } + + return 0; } diff --git a/contrib/libs/liburing/test/pollfree.c b/contrib/libs/liburing/test/pollfree.c index 4ed61e3091..20a8bec60b 100644 --- a/contrib/libs/liburing/test/pollfree.c +++ b/contrib/libs/liburing/test/pollfree.c @@ -253,7 +253,7 @@ static void kill_and_wait(int pid, int* status) } } -static void setup_test() +static void setup_test(void) { prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); setpgrp(); @@ -343,7 +343,7 @@ static void loop(void) #define __NR_io_uring_enter 426 #endif -uint64_t r[4] = {0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x0}; +static uint64_t r[4] = {0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x0}; void execute_call(int call) { @@ -404,13 +404,13 @@ int main(int argc, char *argv[]) if (argc > 1) return 0; - ret = mmap((void *)0x1ffff000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + ret = mmap((void *)0x1ffff000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); if (ret == MAP_FAILED) return 0; - ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + ret = mmap((void *)0x20000000ul, 0x1000000ul, 7ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); if (ret == MAP_FAILED) return 0; - ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + ret = mmap((void *)0x21000000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); if (ret == MAP_FAILED) return 0; loop(); diff --git a/contrib/libs/liburing/test/read-before-exit.c b/contrib/libs/liburing/test/read-before-exit.c index 6510a09263..2fa632c5e1 100644 --- a/contrib/libs/liburing/test/read-before-exit.c +++ b/contrib/libs/liburing/test/read-before-exit.c @@ -15,6 +15,8 @@ #include "liburing.h" #include "helpers.h" +static int no_iopoll; + struct data { struct io_uring *ring; int timer_fd1; @@ -23,7 +25,7 @@ struct data { uint64_t buf2; }; -void *submit(void *data) +static void *submit(void *data) { struct io_uring_sqe *sqe; struct data *d = data; @@ -36,8 +38,21 @@ void *submit(void *data) io_uring_prep_read(sqe, d->timer_fd2, &d->buf2, sizeof(d->buf2), 0); ret = io_uring_submit(d->ring); - if (ret != 2) + if (ret != 2) { + struct io_uring_cqe *cqe; + + /* + * Kernels without submit-all-on-error behavior will + * fail submitting all, check if that's the case and + * don't error + */ + ret = io_uring_peek_cqe(d->ring, &cqe); + if (!ret && cqe->res == -EOPNOTSUPP) { + no_iopoll = 1; + return NULL; + } return (void *) (uintptr_t) 1; + } /* Exit suddenly. */ return NULL; @@ -96,9 +111,11 @@ int main(int argc, char *argv[]) for (i = 0; i < 1000; i++) { ret = test(IORING_SETUP_IOPOLL); if (ret) { - fprintf(stderr, "Test IOPOLL failed\n"); + fprintf(stderr, "Test IOPOLL failed loop %d\n", ret); return ret; } + if (no_iopoll) + break; } for (i = 0; i < 100; i++) { diff --git a/contrib/libs/liburing/test/read-write.c b/contrib/libs/liburing/test/read-write.c index 953b0afc53..2ca1fde1a4 100644 --- a/contrib/libs/liburing/test/read-write.c +++ b/contrib/libs/liburing/test/read-write.c @@ -77,6 +77,8 @@ static int __test_io(const char *file, struct io_uring *ring, int write, fd = open(file, open_flags); if (fd < 0) { + if (errno == EINVAL) + return 0; perror("file open"); goto err; } @@ -636,6 +638,53 @@ static int test_rem_buf(int batch, int sqe_flags) return ret; } +static int test_rem_buf_single(int to_rem) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret, expected; + int bgid = 1; + + if (no_buf_select) + return 0; + + ret = io_uring_queue_init(64, &ring, 0); + if (ret) { + fprintf(stderr, "ring create failed: %d\n", ret); + return 1; + } + + ret = provide_buffers_iovec(&ring, bgid); + if (ret) + return ret; + + expected = (to_rem > BUFFERS) ? BUFFERS : to_rem; + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_remove_buffers(sqe, to_rem, bgid); + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit: %d\n", ret); + return -1; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait_cqe=%d\n", ret); + return 1; + } + if (cqe->res != expected) { + fprintf(stderr, "cqe->res=%d, expected=%d\n", cqe->res, expected); + return 1; + } + io_uring_cqe_seen(&ring, cqe); + + io_uring_queue_exit(&ring); + return ret; +} + static int test_io_link(const char *file) { const int nr_links = 100; @@ -949,6 +998,12 @@ int main(int argc, char *argv[]) } } + ret = test_rem_buf_single(BUFFERS + 1); + if (ret) { + fprintf(stderr, "test_rem_buf_single(BUFFERS + 1) failed\n"); + goto err; + } + if (fname != argv[1]) unlink(fname); return 0; diff --git a/contrib/libs/liburing/test/recv-msgall.c b/contrib/libs/liburing/test/recv-msgall.c index 89b12b7268..e0d94f33b1 100644 --- a/contrib/libs/liburing/test/recv-msgall.c +++ b/contrib/libs/liburing/test/recv-msgall.c @@ -19,14 +19,18 @@ #define MAX_MSG 128 #define HOST "127.0.0.1" static __be16 bind_port; +struct recv_data { + pthread_barrier_t barrier; + int use_recvmsg; + struct msghdr msg; +}; static int recv_prep(struct io_uring *ring, struct iovec *iov, int *sock, - int use_recvmsg) + struct recv_data *rd) { struct sockaddr_in saddr; struct io_uring_sqe *sqe; int sockfd, ret, val; - struct msghdr msg = { }; memset(&saddr, 0, sizeof(saddr)); saddr.sin_family = AF_INET; @@ -48,14 +52,17 @@ static int recv_prep(struct io_uring *ring, struct iovec *iov, int *sock, bind_port = saddr.sin_port; sqe = io_uring_get_sqe(ring); - if (!use_recvmsg) { + if (!rd->use_recvmsg) { io_uring_prep_recv(sqe, sockfd, iov->iov_base, iov->iov_len, MSG_WAITALL); } else { - msg.msg_namelen = sizeof(struct sockaddr_in); - msg.msg_iov = iov; - msg.msg_iovlen = 1; - io_uring_prep_recvmsg(sqe, sockfd, &msg, MSG_WAITALL); + struct msghdr *msg = &rd->msg; + + memset(msg, 0, sizeof(*msg)); + msg->msg_namelen = sizeof(struct sockaddr_in); + msg->msg_iov = iov; + msg->msg_iovlen = 1; + io_uring_prep_recvmsg(sqe, sockfd, msg, MSG_WAITALL); } sqe->user_data = 2; @@ -102,11 +109,6 @@ err: return 1; } -struct recv_data { - pthread_mutex_t mutex; - int use_recvmsg; -}; - static void *recv_fn(void *data) { struct recv_data *rd = data; @@ -121,20 +123,20 @@ static void *recv_fn(void *data) ret = t_create_ring_params(1, &ring, &p); if (ret == T_SETUP_SKIP) { - pthread_mutex_unlock(&rd->mutex); + pthread_barrier_wait(&rd->barrier); ret = 0; goto err; } else if (ret < 0) { - pthread_mutex_unlock(&rd->mutex); + pthread_barrier_wait(&rd->barrier); goto err; } - ret = recv_prep(&ring, &iov, &sock, rd->use_recvmsg); + ret = recv_prep(&ring, &iov, &sock, rd); if (ret) { fprintf(stderr, "recv_prep failed: %d\n", ret); goto err; } - pthread_mutex_unlock(&rd->mutex); + pthread_barrier_wait(&rd->barrier); ret = do_recv(&ring); close(sock); io_uring_queue_exit(&ring); @@ -218,28 +220,24 @@ err: static int test(int use_recvmsg) { - pthread_mutexattr_t attr; pthread_t recv_thread; struct recv_data rd; int ret; void *retval; - pthread_mutexattr_init(&attr); - pthread_mutexattr_setpshared(&attr, 1); - pthread_mutex_init(&rd.mutex, &attr); - pthread_mutex_lock(&rd.mutex); + pthread_barrier_init(&rd.barrier, NULL, 2); rd.use_recvmsg = use_recvmsg; ret = pthread_create(&recv_thread, NULL, recv_fn, &rd); if (ret) { fprintf(stderr, "Thread create failed: %d\n", ret); - pthread_mutex_unlock(&rd.mutex); return 1; } - pthread_mutex_lock(&rd.mutex); + pthread_barrier_wait(&rd.barrier); do_send(); pthread_join(recv_thread, &retval); + pthread_barrier_destroy(&rd.barrier); return (intptr_t)retval; } diff --git a/contrib/libs/liburing/test/recv-multishot.c b/contrib/libs/liburing/test/recv-multishot.c index 67a53567b7..c80fa18547 100644 --- a/contrib/libs/liburing/test/recv-multishot.c +++ b/contrib/libs/liburing/test/recv-multishot.c @@ -10,6 +10,7 @@ #include <sys/types.h> #include <sys/socket.h> #include <pthread.h> +#include <assert.h> #include "liburing.h" #include "helpers.h" @@ -265,11 +266,19 @@ static int test(struct args *args) bool const is_last = i == recv_cqes - 1; + /* + * Older kernels could terminate multishot early due to overflow, + * but later ones will not. So discriminate based on the MORE flag. + */ + bool const early_last = args->early_error == ERROR_EARLY_OVERFLOW && + !args->wait_each && + i == N_CQE_OVERFLOW && + !(cqe->flags & IORING_CQE_F_MORE); + bool const should_be_last = (cqe->res <= 0) || (args->stream && is_last) || - (args->early_error == ERROR_EARLY_OVERFLOW && - !args->wait_each && i == N_CQE_OVERFLOW); + early_last; int *this_recv; int orig_payload_size = cqe->res; @@ -329,7 +338,7 @@ static int test(struct args *args) case ERROR_EARLY_LAST: fprintf(stderr, "bad error_early\n"); goto cleanup; - }; + } if (cqe->res <= 0 && cqe->flags & IORING_CQE_F_BUFFER) { fprintf(stderr, "final BUFFER flag set\n"); @@ -462,6 +471,84 @@ cleanup: return ret; } +static int test_enobuf(void) +{ + struct io_uring ring; + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqes[16]; + char buffs[256]; + int ret, i, fds[2]; + + if (t_create_ring(8, &ring, 0) != T_SETUP_OK) { + fprintf(stderr, "ring create\n"); + return -1; + } + + ret = t_create_socket_pair(fds, false); + if (ret) { + fprintf(stderr, "t_create_socket_pair\n"); + return ret; + } + + sqe = io_uring_get_sqe(&ring); + assert(sqe); + /* deliberately only 2 provided buffers */ + io_uring_prep_provide_buffers(sqe, &buffs[0], 1, 2, 0, 0); + io_uring_sqe_set_data64(sqe, 0); + + sqe = io_uring_get_sqe(&ring); + assert(sqe); + io_uring_prep_recv_multishot(sqe, fds[0], NULL, 0, 0); + io_uring_sqe_set_data64(sqe, 1); + sqe->buf_group = 0; + sqe->flags |= IOSQE_BUFFER_SELECT; + + ret = io_uring_submit(&ring); + if (ret != 2) { + fprintf(stderr, "bad submit %d\n", ret); + return -1; + } + for (i = 0; i < 3; i++) { + do { + ret = write(fds[1], "?", 1); + } while (ret == -1 && errno == EINTR); + } + + ret = io_uring_wait_cqes(&ring, &cqes[0], 4, NULL, NULL); + if (ret) { + fprintf(stderr, "wait cqes\n"); + return ret; + } + + ret = io_uring_peek_batch_cqe(&ring, &cqes[0], 4); + if (ret != 4) { + fprintf(stderr, "peek batch cqes\n"); + return -1; + } + + /* provide buffers */ + assert(cqes[0]->user_data == 0); + assert(cqes[0]->res == 0); + + /* valid recv */ + assert(cqes[1]->user_data == 1); + assert(cqes[2]->user_data == 1); + assert(cqes[1]->res == 1); + assert(cqes[2]->res == 1); + assert(cqes[1]->flags & (IORING_CQE_F_BUFFER | IORING_CQE_F_MORE)); + assert(cqes[2]->flags & (IORING_CQE_F_BUFFER | IORING_CQE_F_MORE)); + + /* missing buffer */ + assert(cqes[3]->user_data == 1); + assert(cqes[3]->res == -ENOBUFS); + assert(!(cqes[3]->flags & (IORING_CQE_F_BUFFER | IORING_CQE_F_MORE))); + + close(fds[0]); + close(fds[1]); + io_uring_queue_exit(&ring); + return 0; +} + int main(int argc, char *argv[]) { int ret; @@ -502,5 +589,11 @@ int main(int argc, char *argv[]) } } + ret = test_enobuf(); + if (ret) { + fprintf(stderr, "test_enobuf() failed: %d\n", ret); + return T_EXIT_FAIL; + } + return T_EXIT_PASS; } diff --git a/contrib/libs/liburing/test/reg-hint.c b/contrib/libs/liburing/test/reg-hint.c new file mode 100644 index 0000000000..fb099d8915 --- /dev/null +++ b/contrib/libs/liburing/test/reg-hint.c @@ -0,0 +1,57 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test alloc hint sanity after unregistering the file table + */ +#include <stdio.h> +#include <unistd.h> +#include <sys/socket.h> + +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct io_uring ring; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + io_uring_queue_init(1, &ring, 0); + + ret = io_uring_register_files_sparse(&ring, 16); + if (ret) { + if (ret == -EINVAL) + return T_EXIT_SKIP; + + fprintf(stderr, "Failed to register file table: %d\n", ret); + return T_EXIT_FAIL; + } + io_uring_unregister_files(&ring); + + sqe = io_uring_get_sqe(&ring); + io_uring_prep_socket_direct_alloc(sqe, AF_UNIX, SOCK_DGRAM, 0, 0); + + ret = io_uring_submit(&ring); + if (ret != 1) { + fprintf(stderr, "submit %d\n", ret); + return T_EXIT_FAIL; + } + + ret = io_uring_wait_cqe(&ring, &cqe); + if (ret) { + fprintf(stderr, "wait cqe: %d\n", ret); + return T_EXIT_FAIL; + } + + if (cqe->res != -ENFILE) { + fprintf(stderr, "Bad CQE res: %d\n", cqe->res); + return T_EXIT_FAIL; + } + + io_uring_cqe_seen(&ring, cqe); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/reg-hint.t/ya.make b/contrib/libs/liburing/test/reg-hint.t/ya.make new file mode 100644 index 0000000000..ef01ee9b41 --- /dev/null +++ b/contrib/libs/liburing/test/reg-hint.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + reg-hint.c +) + +END() diff --git a/contrib/libs/liburing/test/reg-reg-ring.c b/contrib/libs/liburing/test/reg-reg-ring.c new file mode 100644 index 0000000000..2bce730027 --- /dev/null +++ b/contrib/libs/liburing/test/reg-reg-ring.c @@ -0,0 +1,91 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Test io_uring_register with a registered ring (IORING_REGISTER_USE_REGISTERED_RING) + * + */ +#include <stdio.h> + +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + struct io_uring ring; + unsigned values[2]; + int ret; + + if (argc > 1) + return T_EXIT_SKIP; + + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return T_EXIT_FAIL; + } + + if (!(ring.features & IORING_FEAT_REG_REG_RING)) { + fprintf(stderr, "IORING_FEAT_REG_REG_RING not available in kernel\n"); + io_uring_queue_exit(&ring); + return T_EXIT_SKIP; + } + + ret = io_uring_close_ring_fd(&ring); + if (ret != -EINVAL) { + fprintf(stderr, "closing ring fd should EINVAL before register\n"); + goto err; + } + + ret = io_uring_unregister_ring_fd(&ring); + if (ret != -EINVAL) { + fprintf(stderr, "unregistering not-registered ring fd should fail\n"); + goto err; + } + + ret = io_uring_register_ring_fd(&ring); + if (ret != 1) { + fprintf(stderr, "registering ring fd failed\n"); + goto err; + } + + ret = io_uring_register_ring_fd(&ring); + if (ret != -EEXIST) { + fprintf(stderr, "registering already-registered ring fd should fail\n"); + goto err; + } + + /* Test a simple io_uring_register operation expected to work. + * io_uring_register_iowq_max_workers is arbitrary. + */ + values[0] = values[1] = 0; + ret = io_uring_register_iowq_max_workers(&ring, values); + if (ret || (values[0] == 0 && values[1] == 0)) { + fprintf(stderr, "io_uring_register operation failed before closing ring fd\n"); + goto err; + } + + ret = io_uring_close_ring_fd(&ring); + if (ret != 1) { + fprintf(stderr, "closing ring fd failed\n"); + goto err; + } + + values[0] = values[1] = 0; + ret = io_uring_register_iowq_max_workers(&ring, values); + if (ret || (values[0] == 0 && values[1] == 0)) { + fprintf(stderr, "io_uring_register operation failed after closing ring fd\n"); + goto err; + } + + ret = io_uring_close_ring_fd(&ring); + if (ret != -EBADF) { + fprintf(stderr, "closing already-closed ring fd should fail\n"); + goto err; + } + + io_uring_queue_exit(&ring); + return T_EXIT_PASS; + +err: + io_uring_queue_exit(&ring); + return T_EXIT_FAIL; +} diff --git a/contrib/libs/liburing/test/reg-reg-ring.t/ya.make b/contrib/libs/liburing/test/reg-reg-ring.t/ya.make new file mode 100644 index 0000000000..ddb58c45f2 --- /dev/null +++ b/contrib/libs/liburing/test/reg-reg-ring.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + reg-reg-ring.c +) + +END() diff --git a/contrib/libs/liburing/test/regbuf-merge.c b/contrib/libs/liburing/test/regbuf-merge.c new file mode 100644 index 0000000000..bb2e1286cb --- /dev/null +++ b/contrib/libs/liburing/test/regbuf-merge.c @@ -0,0 +1,92 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +// autogenerated by syzkaller (https://github.com/google/syzkaller) + +#include <endian.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <unistd.h> + +#include "helpers.h" + +#ifndef __NR_io_uring_register +#define __NR_io_uring_register 427 +#endif +#ifndef __NR_io_uring_setup +#define __NR_io_uring_setup 425 +#endif + +#define SIZEOF_IO_URING_SQE 64 +#define SIZEOF_IO_URING_CQE 16 +#define SQ_HEAD_OFFSET 0 +#define SQ_TAIL_OFFSET 64 +#define SQ_RING_MASK_OFFSET 256 +#define SQ_RING_ENTRIES_OFFSET 264 +#define SQ_FLAGS_OFFSET 276 +#define SQ_DROPPED_OFFSET 272 +#define CQ_HEAD_OFFSET 128 +#define CQ_TAIL_OFFSET 192 +#define CQ_RING_MASK_OFFSET 260 +#define CQ_RING_ENTRIES_OFFSET 268 +#define CQ_RING_OVERFLOW_OFFSET 284 +#define CQ_FLAGS_OFFSET 280 +#define CQ_CQES_OFFSET 320 + +static long syz_io_uring_setup(volatile long a0, volatile long a1, volatile long a2, volatile long a3, volatile long a4, volatile long a5) +{ + uint32_t entries = (uint32_t)a0; + struct io_uring_params* setup_params = (struct io_uring_params*)a1; + void* vma1 = (void*)a2; + void* vma2 = (void*)a3; + void** ring_ptr_out = (void**)a4; + void** sqes_ptr_out = (void**)a5; + uint32_t fd_io_uring = syscall(__NR_io_uring_setup, entries, setup_params); + uint32_t sq_ring_sz = setup_params->sq_off.array + setup_params->sq_entries * sizeof(uint32_t); + uint32_t cq_ring_sz = setup_params->cq_off.cqes + setup_params->cq_entries * SIZEOF_IO_URING_CQE; + uint32_t ring_sz = sq_ring_sz > cq_ring_sz ? sq_ring_sz : cq_ring_sz; + *ring_ptr_out = mmap(vma1, ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, IORING_OFF_SQ_RING); + uint32_t sqes_sz = setup_params->sq_entries * SIZEOF_IO_URING_SQE; + *sqes_ptr_out = mmap(vma2, sqes_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd_io_uring, IORING_OFF_SQES); + return fd_io_uring; +} + +static uint64_t r[1] = {0xffffffffffffffff}; + +int main(int argc, char *argv[]) +{ + intptr_t res = 0; + + if (argc > 1) + return T_EXIT_SKIP; + + mmap((void *) 0x1ffff000ul, 0x1000ul, PROT_NONE, + MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0ul); + mmap((void *) 0x20000000ul, 0x1000000ul, PROT_READ|PROT_WRITE|PROT_EXEC, + MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0ul); + mmap((void *) 0x21000000ul, 0x1000ul, PROT_NONE, + MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0ul); + + *(uint32_t*)0x20000684 = 0; + *(uint32_t*)0x20000688 = 0; + *(uint32_t*)0x2000068c = 0; + *(uint32_t*)0x20000690 = 0; + *(uint32_t*)0x20000698 = -1; + memset((void*)0x2000069c, 0, 12); + + res = syz_io_uring_setup(0x2fd6, 0x20000680, 0x20ffd000, 0x20ffc000, + 0x20000700, 0x20000740); + if (res != -1) + r[0] = res; + + *(uint64_t*)0x20002840 = 0; + *(uint64_t*)0x20002848 = 0; + *(uint64_t*)0x20002850 = 0x20000840; + *(uint64_t*)0x20002858 = 0x1000; + syscall(__NR_io_uring_register, r[0], 0ul, 0x20002840ul, 2ul); + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/regbuf-merge.t/ya.make b/contrib/libs/liburing/test/regbuf-merge.t/ya.make new file mode 100644 index 0000000000..9226844487 --- /dev/null +++ b/contrib/libs/liburing/test/regbuf-merge.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + regbuf-merge.c +) + +END() diff --git a/contrib/libs/liburing/test/rename.c b/contrib/libs/liburing/test/rename.c index c72a36ea1b..d11ad17b33 100644 --- a/contrib/libs/liburing/test/rename.c +++ b/contrib/libs/liburing/test/rename.c @@ -1,7 +1,7 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ /* - * Description: run various nop tests + * Description: run various rename tests * */ #include <errno.h> diff --git a/contrib/libs/liburing/test/ring-leak.c b/contrib/libs/liburing/test/ring-leak.c index 26338a48f1..e052802314 100644 --- a/contrib/libs/liburing/test/ring-leak.c +++ b/contrib/libs/liburing/test/ring-leak.c @@ -242,7 +242,6 @@ int main(int argc, char *argv[]) update); return 1; } - break; } if (socketpair(AF_UNIX, SOCK_DGRAM, 0, sp) != 0) { diff --git a/contrib/libs/liburing/test/ring-leak2.c b/contrib/libs/liburing/test/ring-leak2.c index b0b43413ef..4a33f1f440 100644 --- a/contrib/libs/liburing/test/ring-leak2.c +++ b/contrib/libs/liburing/test/ring-leak2.c @@ -47,7 +47,7 @@ static struct io_uring *client_ring; static int client_eventfd = -1; -int setup_io_uring(struct io_uring *ring) +static int setup_io_uring(struct io_uring *ring) { struct io_uring_params p = { }; int ret; diff --git a/contrib/libs/liburing/test/ringbuf-read.c b/contrib/libs/liburing/test/ringbuf-read.c index 2eede18213..5ff6738b04 100644 --- a/contrib/libs/liburing/test/ringbuf-read.c +++ b/contrib/libs/liburing/test/ringbuf-read.c @@ -38,7 +38,6 @@ static int verify_buffer(char *buf, char val) static int test(const char *filename, int dio, int async) { - struct io_uring_buf_reg reg = { }; struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; struct io_uring ring; @@ -53,10 +52,13 @@ static int test(const char *filename, int dio, int async) return 1; } - if (dio) + if (dio) { fd = open(filename, O_DIRECT | O_RDONLY); - else + if (fd < 0 && errno == EINVAL) + return T_EXIT_SKIP; + } else { fd = open(filename, O_RDONLY); + } if (fd < 0) { perror("open"); return 1; @@ -66,15 +68,9 @@ static int test(const char *filename, int dio, int async) if (posix_memalign((void **) &buf, 4096, FSIZE)) return 1; - if (posix_memalign((void **) &br, 4096, 4096)) - return 1; - - reg.ring_addr = (unsigned long) br; - reg.ring_entries = NR_BUFS; - reg.bgid = 1; - ret = io_uring_register_buf_ring(&ring, ®, 0); - if (ret) { + br = io_uring_setup_buf_ring(&ring, NR_BUFS, 1, 0, &ret); + if (!br) { if (ret == -EINVAL) { no_buf_ring = 1; return 0; @@ -164,7 +160,7 @@ int main(int argc, char *argv[]) close(fd); ret = test(fname, 1, 0); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "dio test failed\n"); goto err; } @@ -178,13 +174,13 @@ int main(int argc, char *argv[]) } ret = test(fname, 1, 1); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "dio async test failed\n"); goto err; } ret = test(fname, 0, 1); - if (ret) { + if (ret == T_EXIT_FAIL) { fprintf(stderr, "buffered async test failed\n"); goto err; } diff --git a/contrib/libs/liburing/test/send-zerocopy.c b/contrib/libs/liburing/test/send-zerocopy.c index b201e68317..27300f9209 100644 --- a/contrib/libs/liburing/test/send-zerocopy.c +++ b/contrib/libs/liburing/test/send-zerocopy.c @@ -5,7 +5,6 @@ #include <stdint.h> #include <assert.h> #include <errno.h> -#include <error.h> #include <limits.h> #include <fcntl.h> #include <unistd.h> @@ -13,7 +12,6 @@ #include <string.h> #include <arpa/inet.h> -#include <linux/errqueue.h> #include <linux/if_packet.h> #include <linux/ipv6.h> #include <linux/socket.h> @@ -35,16 +33,18 @@ #include <sys/time.h> #include <sys/types.h> #include <sys/wait.h> +#include <sys/mman.h> +#include <linux/mman.h> #include "liburing.h" #include "helpers.h" #define MAX_MSG 128 -#define PORT 10200 #define HOST "127.0.0.1" #define HOSTV6 "::1" +#define MAX_IOV 32 #define CORK_REQS 5 #define RX_TAG 10000 #define BUFFER_OFFSET 41 @@ -58,10 +58,17 @@ enum { BUF_T_SMALL, BUF_T_NONALIGNED, BUF_T_LARGE, + BUF_T_HUGETLB, + + __BUF_NR, }; +/* 32MB, should be enough to trigger a short send */ +#define LARGE_BUF_SIZE (1U << 25) + +static size_t page_sz; static char *tx_buffer, *rx_buffer; -static struct iovec buffers_iov[4]; +static struct iovec buffers_iov[__BUF_NR]; static bool has_sendmsg; static bool check_cq_empty(struct io_uring *ring) @@ -116,43 +123,72 @@ static int test_basic_send(struct io_uring *ring, int sock_tx, int sock_rx) return T_EXIT_PASS; } -static int test_send_faults(struct io_uring *ring, int sock_tx, int sock_rx) +static int test_send_faults(int sock_tx, int sock_rx) { struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; int msg_flags = 0; unsigned zc_flags = 0; int payload_size = 100; - int ret, i, nr_cqes = 2; + int ret, i, nr_cqes, nr_reqs = 3; + struct io_uring ring; - sqe = io_uring_get_sqe(ring); + ret = io_uring_queue_init(32, &ring, IORING_SETUP_SUBMIT_ALL); + if (ret) { + fprintf(stderr, "queue init failed: %d\n", ret); + return -1; + } + + /* invalid buffer */ + sqe = io_uring_get_sqe(&ring); io_uring_prep_send_zc(sqe, sock_tx, (void *)1UL, payload_size, msg_flags, zc_flags); sqe->user_data = 1; - sqe = io_uring_get_sqe(ring); + /* invalid address */ + sqe = io_uring_get_sqe(&ring); io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, payload_size, msg_flags, zc_flags); - sqe->user_data = 2; io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)1UL, sizeof(struct sockaddr_in6)); + sqe->user_data = 2; - ret = io_uring_submit(ring); - assert(ret == 2); + /* invalid send/recv flags */ + sqe = io_uring_get_sqe(&ring); + io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, payload_size, + msg_flags, ~0U); + sqe->user_data = 3; + ret = io_uring_submit(&ring); + assert(ret == nr_reqs); + + nr_cqes = nr_reqs; for (i = 0; i < nr_cqes; i++) { - ret = io_uring_wait_cqe(ring, &cqe); + ret = io_uring_wait_cqe(&ring, &cqe); assert(!ret); - assert(cqe->user_data <= 2); + assert(cqe->user_data <= nr_reqs); if (!(cqe->flags & IORING_CQE_F_NOTIF)) { - assert(cqe->res == -EFAULT); + int expected = (cqe->user_data == 3) ? -EINVAL : -EFAULT; + + if (cqe->res != expected) { + fprintf(stderr, "invalid cqe res %i vs expected %i, " + "user_data %i\n", + cqe->res, expected, (int)cqe->user_data); + return -1; + } if (cqe->flags & IORING_CQE_F_MORE) nr_cqes++; + } else { + if (cqe->res != 0 || cqe->flags != IORING_CQE_F_NOTIF) { + fprintf(stderr, "invalid notif cqe %i %i\n", + cqe->res, cqe->flags); + return -1; + } } - io_uring_cqe_seen(ring, cqe); + io_uring_cqe_seen(&ring, cqe); } - assert(check_cq_empty(ring)); + assert(check_cq_empty(&ring)); return T_EXIT_PASS; } @@ -161,10 +197,9 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, bool ipv6, bool client_connect, bool msg_zc, bool tcp) { - int family, addr_size; - int ret, val; - int listen_sock = -1; - int sock; + socklen_t addr_size; + int family, sock, listen_sock = -1; + int ret; memset(addr, 0, sizeof(*addr)); if (ipv6) { @@ -172,14 +207,14 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, family = AF_INET6; saddr->sin6_family = family; - saddr->sin6_port = htons(PORT); + saddr->sin6_port = htons(0); addr_size = sizeof(*saddr); } else { struct sockaddr_in *saddr = (struct sockaddr_in *)addr; family = AF_INET; saddr->sin_family = family; - saddr->sin_port = htons(PORT); + saddr->sin_port = htons(0); saddr->sin_addr.s_addr = htonl(INADDR_ANY); addr_size = sizeof(*saddr); } @@ -194,16 +229,19 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, perror("socket"); return 1; } - val = 1; - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); - val = 1; - setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &val, sizeof(val)); ret = bind(sock, (struct sockaddr *)addr, addr_size); if (ret < 0) { perror("bind"); return 1; } + + ret = getsockname(sock, (struct sockaddr *)addr, &addr_size); + if (ret < 0) { + fprintf(stderr, "getsockname failed %i\n", errno); + return 1; + } + if (tcp) { ret = listen(sock, 128); assert(ret != -1); @@ -238,11 +276,17 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, } } if (msg_zc) { - val = 1; +#ifdef SO_ZEROCOPY + int val = 1; + if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) { perror("setsockopt zc"); return 1; } +#else + fprintf(stderr, "no SO_ZEROCOPY\n"); + return 1; +#endif } if (tcp) { *sock_server = accept(listen_sock, NULL, NULL); @@ -255,25 +299,40 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, return 0; } +struct send_conf { + bool fixed_buf; + bool mix_register; + bool cork; + bool force_async; + bool use_sendmsg; + bool tcp; + bool zc; + bool iovec; + bool long_iovec; + bool poll_first; + int buf_index; + struct sockaddr_storage *addr; +}; + static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_server, - bool fixed_buf, struct sockaddr_storage *addr, - bool cork, bool mix_register, - int buf_idx, bool force_async, bool use_sendmsg) + struct send_conf *conf) { - struct iovec iov[CORK_REQS]; + struct iovec iov[MAX_IOV]; struct msghdr msghdr[CORK_REQS]; const unsigned zc_flags = 0; struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; - int nr_reqs = cork ? CORK_REQS : 1; + int nr_reqs = conf->cork ? CORK_REQS : 1; int i, ret, nr_cqes, addr_len = 0; - size_t send_size = buffers_iov[buf_idx].iov_len; + size_t send_size = buffers_iov[conf->buf_index].iov_len; size_t chunk_size = send_size / nr_reqs; size_t chunk_size_last = send_size - chunk_size * (nr_reqs - 1); - char *buf = buffers_iov[buf_idx].iov_base; + char *buf = buffers_iov[conf->buf_index].iov_base; - if (addr) { - sa_family_t fam = ((struct sockaddr_in *)addr)->sin_family; + assert(MAX_IOV >= CORK_REQS); + + if (conf->addr) { + sa_family_t fam = ((struct sockaddr_in *)conf->addr)->sin_family; addr_len = (fam == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); @@ -282,46 +341,87 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se memset(rx_buffer, 0, send_size); for (i = 0; i < nr_reqs; i++) { - bool real_fixed_buf = fixed_buf; + bool real_fixed_buf = conf->fixed_buf; size_t cur_size = chunk_size; int msg_flags = MSG_WAITALL; - if (mix_register) + if (conf->mix_register) real_fixed_buf = rand() & 1; - if (cork && i != nr_reqs - 1) + if (i != nr_reqs - 1) msg_flags |= MSG_MORE; - if (i == nr_reqs - 1) + else cur_size = chunk_size_last; sqe = io_uring_get_sqe(ring); - if (!use_sendmsg) { - io_uring_prep_send_zc(sqe, sock_client, buf + i * chunk_size, - cur_size, msg_flags, zc_flags); + if (!conf->use_sendmsg) { + if (conf->zc) { + io_uring_prep_send_zc(sqe, sock_client, buf + i * chunk_size, + cur_size, msg_flags, zc_flags); + } else { + io_uring_prep_send(sqe, sock_client, buf + i * chunk_size, + cur_size, msg_flags); + } + if (real_fixed_buf) { sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; - sqe->buf_index = buf_idx; + sqe->buf_index = conf->buf_index; } - if (addr) - io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)addr, + if (conf->addr) + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)conf->addr, addr_len); } else { - io_uring_prep_sendmsg_zc(sqe, sock_client, &msghdr[i], msg_flags); + struct iovec *io; + int iov_len; + + if (conf->zc) + io_uring_prep_sendmsg_zc(sqe, sock_client, &msghdr[i], msg_flags); + else + io_uring_prep_sendmsg(sqe, sock_client, &msghdr[i], msg_flags); + + if (!conf->iovec) { + io = &iov[i]; + iov_len = 1; + iov[i].iov_len = cur_size; + iov[i].iov_base = buf + i * chunk_size; + } else { + char *it = buf; + int j; + + assert(nr_reqs == 1); + iov_len = conf->long_iovec ? MAX_IOV : 4; + io = iov; + + for (j = 0; j < iov_len; j++) + io[j].iov_len = 1; + /* first want to be easily advanced */ + io[0].iov_base = it; + it += io[0].iov_len; + /* this should cause retry */ + io[1].iov_len = chunk_size - iov_len + 1; + io[1].iov_base = it; + it += io[1].iov_len; + /* fill the rest */ + for (j = 2; j < iov_len; j++) { + io[j].iov_base = it; + it += io[j].iov_len; + } + } memset(&msghdr[i], 0, sizeof(msghdr[i])); - iov[i].iov_len = cur_size; - iov[i].iov_base = buf + i * chunk_size; - msghdr[i].msg_iov = &iov[i]; - msghdr[i].msg_iovlen = 1; - if (addr) { - msghdr[i].msg_name = addr; + msghdr[i].msg_iov = io; + msghdr[i].msg_iovlen = iov_len; + if (conf->addr) { + msghdr[i].msg_name = conf->addr; msghdr[i].msg_namelen = addr_len; } } sqe->user_data = i; - if (force_async) + if (conf->force_async) sqe->flags |= IOSQE_ASYNC; + if (conf->poll_first) + sqe->ioprio |= IORING_RECVSEND_POLL_FIRST; if (i != nr_reqs - 1) sqe->flags |= IOSQE_IO_LINK; } @@ -336,7 +436,7 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se return 1; } - nr_cqes = 2 * nr_reqs + 1; + nr_cqes = nr_reqs + 1; for (i = 0; i < nr_cqes; i++) { int expected = chunk_size; @@ -347,19 +447,26 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se } if (cqe->user_data == RX_TAG) { if (cqe->res != send_size) { - fprintf(stderr, "rx failed %i\n", cqe->res); + fprintf(stderr, "rx failed res: %i, expected %i\n", + cqe->res, (int)send_size); return 1; } io_uring_cqe_seen(ring, cqe); continue; } - + if ((cqe->flags & IORING_CQE_F_MORE) && (cqe->flags & IORING_CQE_F_NOTIF)) { + fprintf(stderr, "unexpected cflags %i res %i\n", + cqe->flags, cqe->res); + return 1; + } if (cqe->user_data >= nr_reqs) { fprintf(stderr, "invalid user_data %lu\n", (unsigned long)cqe->user_data); return 1; } if (!(cqe->flags & IORING_CQE_F_NOTIF)) { + if (cqe->flags & IORING_CQE_F_MORE) + nr_cqes++; if (cqe->user_data == nr_reqs - 1) expected = chunk_size_last; if (cqe->res != expected) { @@ -368,12 +475,6 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se return 1; } } - if ((cqe->flags & IORING_CQE_F_MORE) == - (cqe->flags & IORING_CQE_F_NOTIF)) { - fprintf(stderr, "unexpected cflags %i res %i\n", - cqe->flags, cqe->res); - return 1; - } io_uring_cqe_seen(ring, cqe); } @@ -389,57 +490,96 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se static int test_inet_send(struct io_uring *ring) { + struct send_conf conf; struct sockaddr_storage addr; int sock_client = -1, sock_server = -1; int ret, j, i; + int buf_index; - for (j = 0; j < 16; j++) { + for (j = 0; j < 32; j++) { bool ipv6 = j & 1; bool client_connect = j & 2; bool msg_zc_set = j & 4; bool tcp = j & 8; + bool swap_sockets = j & 16; if (tcp && !client_connect) continue; - + if (swap_sockets && !tcp) + continue; +#ifndef SO_ZEROCOPY + if (msg_zc_set) + continue; +#endif ret = create_socketpair_ip(&addr, &sock_client, &sock_server, ipv6, client_connect, msg_zc_set, tcp); if (ret) { fprintf(stderr, "sock prep failed %d\n", ret); return 1; } + if (swap_sockets) { + int tmp_sock = sock_client; - for (i = 0; i < 256; i++) { - int buf_flavour = i & 3; - bool fixed_buf = i & 4; - struct sockaddr_storage *addr_arg = (i & 8) ? &addr : NULL; - bool cork = i & 16; - bool mix_register = i & 32; - bool force_async = i & 64; - bool use_sendmsg = i & 128; + sock_client = sock_server; + sock_server = tmp_sock; + } - if (buf_flavour == BUF_T_LARGE && !tcp) + for (i = 0; i < 1024; i++) { + bool regbuf; + + conf.use_sendmsg = i & 1; + conf.poll_first = i & 2; + conf.fixed_buf = i & 4; + conf.addr = (i & 8) ? &addr : NULL; + conf.cork = i & 16; + conf.mix_register = i & 32; + conf.force_async = i & 64; + conf.zc = i & 128; + conf.iovec = i & 256; + conf.long_iovec = i & 512; + conf.tcp = tcp; + regbuf = conf.mix_register || conf.fixed_buf; + + if (conf.iovec && (!conf.use_sendmsg || regbuf || conf.cork)) continue; - if (!buffers_iov[buf_flavour].iov_base) + if (!conf.zc) { + if (regbuf) + continue; + /* + * Non zerocopy send w/ addr was added together with sendmsg_zc, + * skip if we the kernel doesn't support it. + */ + if (conf.addr && !has_sendmsg) + continue; + } + if (tcp && (conf.cork || conf.addr)) continue; - if (tcp && (cork || addr_arg)) + if (conf.mix_register && (!conf.cork || conf.fixed_buf)) continue; - if (mix_register && (!cork || fixed_buf)) + if (!client_connect && conf.addr == NULL) continue; - if (!client_connect && addr_arg == NULL) + if (conf.use_sendmsg && (regbuf || !has_sendmsg)) continue; - if (use_sendmsg && (mix_register || fixed_buf || !has_sendmsg)) + if (msg_zc_set && !conf.zc) continue; - ret = do_test_inet_send(ring, sock_client, sock_server, fixed_buf, - addr_arg, cork, mix_register, - buf_flavour, force_async, use_sendmsg); - if (ret) { - fprintf(stderr, "send failed fixed buf %i, conn %i, addr %i, " - "cork %i\n", - fixed_buf, client_connect, !!addr_arg, - cork); - return 1; + for (buf_index = 0; buf_index < ARRAY_SIZE(buffers_iov); buf_index++) { + size_t len = buffers_iov[buf_index].iov_len; + + if (!buffers_iov[buf_index].iov_base) + continue; + if (!tcp && len > 4 * page_sz) + continue; + + conf.buf_index = buf_index; + ret = do_test_inet_send(ring, sock_client, sock_server, &conf); + if (ret) { + fprintf(stderr, "send failed fixed buf %i, " + "conn %i, addr %i, cork %i\n", + conf.fixed_buf, client_connect, + !!conf.addr, conf.cork); + return 1; + } } } @@ -589,6 +729,8 @@ int main(int argc, char *argv[]) if (argc > 1) return T_EXIT_SKIP; + page_sz = sysconf(_SC_PAGESIZE); + /* create TCP IPv6 pair */ ret = create_socketpair_ip(&addr, &sp[0], &sp[1], true, true, false, true); if (ret) { @@ -596,30 +738,54 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } - len = 1U << 25; /* 32MB, should be enough to trigger a short send */ - tx_buffer = aligned_alloc(4096, len); - rx_buffer = aligned_alloc(4096, len); + len = LARGE_BUF_SIZE; + tx_buffer = aligned_alloc(page_sz, len); + rx_buffer = aligned_alloc(page_sz, len); if (tx_buffer && rx_buffer) { buffers_iov[BUF_T_LARGE].iov_base = tx_buffer; buffers_iov[BUF_T_LARGE].iov_len = len; } else { + if (tx_buffer) + free(tx_buffer); + if (rx_buffer) + free(rx_buffer); + printf("skip large buffer tests, can't alloc\n"); - len = 8192; - tx_buffer = aligned_alloc(4096, len); - rx_buffer = aligned_alloc(4096, len); + len = 2 * page_sz; + tx_buffer = aligned_alloc(page_sz, len); + rx_buffer = aligned_alloc(page_sz, len); } if (!tx_buffer || !rx_buffer) { fprintf(stderr, "can't allocate buffers\n"); return T_EXIT_FAIL; } - buffers_iov[BUF_T_NORMAL].iov_base = tx_buffer + 4096; - buffers_iov[BUF_T_NORMAL].iov_len = 4096; + srand((unsigned)time(NULL)); + for (i = 0; i < len; i++) + tx_buffer[i] = i; + memset(rx_buffer, 0, len); + + buffers_iov[BUF_T_NORMAL].iov_base = tx_buffer + page_sz; + buffers_iov[BUF_T_NORMAL].iov_len = page_sz; buffers_iov[BUF_T_SMALL].iov_base = tx_buffer; buffers_iov[BUF_T_SMALL].iov_len = 137; buffers_iov[BUF_T_NONALIGNED].iov_base = tx_buffer + BUFFER_OFFSET; - buffers_iov[BUF_T_NONALIGNED].iov_len = 8192 - BUFFER_OFFSET - 13; + buffers_iov[BUF_T_NONALIGNED].iov_len = 2 * page_sz - BUFFER_OFFSET - 13; + + if (len == LARGE_BUF_SIZE) { + void *huge_page; + int off = page_sz + 27; + + len = 1U << 22; + huge_page = mmap(NULL, len, PROT_READ|PROT_WRITE, + MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS, + -1, 0); + if (huge_page != MAP_FAILED) { + buffers_iov[BUF_T_HUGETLB].iov_base = huge_page + off; + buffers_iov[BUF_T_HUGETLB].iov_len = len - off; + } + } ret = io_uring_queue_init(32, &ring, 0); if (ret) { @@ -627,11 +793,6 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } - srand((unsigned)time(NULL)); - for (i = 0; i < len; i++) - tx_buffer[i] = i; - memset(rx_buffer, 0, len); - ret = test_basic_send(&ring, sp[0], sp[1]); if (ret == T_EXIT_SKIP) return ret; @@ -642,7 +803,7 @@ int main(int argc, char *argv[]) has_sendmsg = io_check_zc_sendmsg(&ring); - ret = test_send_faults(&ring, sp[0], sp[1]); + ret = test_send_faults(sp[0], sp[1]); if (ret) { fprintf(stderr, "test_send_faults() failed\n"); return T_EXIT_FAIL; @@ -672,6 +833,15 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } + if (buffers_iov[BUF_T_HUGETLB].iov_base) { + buffers_iov[BUF_T_HUGETLB].iov_base += 13; + buffers_iov[BUF_T_HUGETLB].iov_len -= 26; + } + if (buffers_iov[BUF_T_LARGE].iov_base) { + buffers_iov[BUF_T_LARGE].iov_base += 13; + buffers_iov[BUF_T_LARGE].iov_len -= 26; + } + ret = test_inet_send(&ring); if (ret) { fprintf(stderr, "test_inet_send() failed\n"); diff --git a/contrib/libs/liburing/test/send_recv.c b/contrib/libs/liburing/test/send_recv.c index 5200494a6c..05b2ffa902 100644 --- a/contrib/libs/liburing/test/send_recv.c +++ b/contrib/libs/liburing/test/send_recv.c @@ -193,13 +193,13 @@ static int do_send(void) sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd < 0) { perror("socket"); - return 1; + goto err2; } ret = connect(sockfd, (struct sockaddr *)&saddr, sizeof(saddr)); if (ret < 0) { perror("connect"); - return 1; + goto err; } sqe = io_uring_get_sqe(&ring); @@ -215,8 +215,7 @@ static int do_send(void) ret = io_uring_wait_cqe(&ring, &cqe); if (cqe->res == -EINVAL) { fprintf(stdout, "send not supported, skipping\n"); - close(sockfd); - return 0; + goto err; } if (cqe->res != iov.iov_len) { fprintf(stderr, "failed cqe: %d\n", cqe->res); @@ -224,9 +223,13 @@ static int do_send(void) } close(sockfd); + io_uring_queue_exit(&ring); return 0; + err: close(sockfd); +err2: + io_uring_queue_exit(&ring); return 1; } diff --git a/contrib/libs/liburing/test/send_recvmsg.c b/contrib/libs/liburing/test/send_recvmsg.c index da64a0fb06..bea40c0796 100644 --- a/contrib/libs/liburing/test/send_recvmsg.c +++ b/contrib/libs/liburing/test/send_recvmsg.c @@ -185,22 +185,11 @@ static void *recv_fn(void *data) if ((rd->buf_ring || rd->buf_select) && !rd->no_buf_add) { if (rd->buf_ring) { - struct io_uring_buf_reg reg = { }; - void *ptr; - - if (posix_memalign(&ptr, 4096, 4096)) - goto err; - - reg.ring_addr = (unsigned long) ptr; - reg.ring_entries = 1; - reg.bgid = BUF_BGID; - if (io_uring_register_buf_ring(&ring, ®, 0)) { + br = io_uring_setup_buf_ring(&ring, 1, BUF_BGID, 0, &ret); + if (!br) { no_pbuf_ring = 1; goto out; } - - br = ptr; - io_uring_buf_ring_init(br); io_uring_buf_ring_add(br, buf, sizeof(buf), BUF_BID, io_uring_buf_ring_mask(1), 0); io_uring_buf_ring_advance(br, 1); @@ -247,9 +236,9 @@ static void *recv_fn(void *data) ret = do_recvmsg(&ring, buf, rd); close(sockfd); - io_uring_queue_exit(&ring); if (br) - free(br); + io_uring_free_buf_ring(&ring, br, 1, BUF_BGID); + io_uring_queue_exit(&ring); err: return (void *)(intptr_t)ret; out: @@ -257,7 +246,7 @@ out: out_no_ring: pthread_mutex_unlock(mutex); if (br) - free(br); + io_uring_free_buf_ring(&ring, br, 1, BUF_BGID); return NULL; } diff --git a/contrib/libs/liburing/test/sendmsg_fs_cve.c b/contrib/libs/liburing/test/sendmsg_fs_cve.c index 3829a5c085..786e7dc551 100644 --- a/contrib/libs/liburing/test/sendmsg_fs_cve.c +++ b/contrib/libs/liburing/test/sendmsg_fs_cve.c @@ -1,5 +1,5 @@ #include "../config-host.h" -/* SPDX-License-Identifier: MIT */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * repro-CVE-2020-29373 -- Reproducer for CVE-2020-29373. * @@ -17,7 +17,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program; if not, see <http://www.gnu.org/licenses/>. + * along with this program; if not, see <https://www.gnu.org/licenses/>. */ #include <unistd.h> diff --git a/contrib/libs/liburing/test/sendmsg_fs_cve.t/ya.make b/contrib/libs/liburing/test/sendmsg_fs_cve.t/ya.make index cb23a9fe65..94e6887afa 100644 --- a/contrib/libs/liburing/test/sendmsg_fs_cve.t/ya.make +++ b/contrib/libs/liburing/test/sendmsg_fs_cve.t/ya.make @@ -2,10 +2,15 @@ PROGRAM() -LICENSE(GPL-2.0-or-later) +LICENSE( + GPL-2.0-or-later AND + GPL-3.0-or-later +) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) +WITHOUT_LICENSE_TEXTS() + PEERDIR( contrib/libs/liburing ) diff --git a/contrib/libs/liburing/test/single-issuer.c b/contrib/libs/liburing/test/single-issuer.c index 22f104e94e..7d12fc1b17 100644 --- a/contrib/libs/liburing/test/single-issuer.c +++ b/contrib/libs/liburing/test/single-issuer.c @@ -6,7 +6,6 @@ #include <stdlib.h> #include <string.h> #include <fcntl.h> -#include <error.h> #include <sys/types.h> #include <sys/wait.h> @@ -57,13 +56,13 @@ static int try_submit(struct io_uring *ring) return ret; if (ret != 1) - error(1, ret, "submit %i", ret); + t_error(1, ret, "submit %i", ret); ret = io_uring_wait_cqe(ring, &cqe); if (ret) - error(1, ret, "wait fail %i", ret); + t_error(1, ret, "wait fail %i", ret); if (cqe->res || cqe->user_data != 42) - error(1, ret, "invalid cqe"); + t_error(1, ret, "invalid cqe"); io_uring_cqe_seen(ring, cqe); return 0; @@ -79,7 +78,6 @@ int main(int argc, char *argv[]) ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER); if (ret == -EINVAL) { - fprintf(stderr, "SETUP_SINGLE_ISSUER is not supported, skip\n"); return T_EXIT_SKIP; } else if (ret) { fprintf(stderr, "io_uring_queue_init() failed %i\n", ret); @@ -107,7 +105,7 @@ int main(int argc, char *argv[]) ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_R_DISABLED); if (ret) - error(1, ret, "ring init (2) %i", ret); + t_error(1, ret, "ring init (2) %i", ret); if (!fork_t()) { io_uring_enable_rings(&ring); @@ -123,7 +121,7 @@ int main(int argc, char *argv[]) ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_R_DISABLED); if (ret) - error(1, ret, "ring init (3) %i", ret); + t_error(1, ret, "ring init (3) %i", ret); io_uring_enable_rings(&ring); if (!fork_t()) { @@ -138,7 +136,7 @@ int main(int argc, char *argv[]) /* test that anyone can submit to a SQPOLL|SINGLE_ISSUER ring */ ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_SQPOLL); if (ret) - error(1, ret, "ring init (4) %i", ret); + t_error(1, ret, "ring init (4) %i", ret); ret = try_submit(&ring); if (ret) { @@ -158,7 +156,7 @@ int main(int argc, char *argv[]) /* test that IORING_ENTER_REGISTERED_RING doesn't break anything */ ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER); if (ret) - error(1, ret, "ring init (5) %i", ret); + t_error(1, ret, "ring init (5) %i", ret); if (!fork_t()) { ret = try_submit(&ring); diff --git a/contrib/libs/liburing/test/skip-cqe.c b/contrib/libs/liburing/test/skip-cqe.c index c73e2b8dea..87010a4ca0 100644 --- a/contrib/libs/liburing/test/skip-cqe.c +++ b/contrib/libs/liburing/test/skip-cqe.c @@ -9,6 +9,7 @@ #include <assert.h> #include "liburing.h" +#include "helpers.h" #define LINK_SIZE 6 #define TIMEOUT_USER_DATA (-1) @@ -324,10 +325,8 @@ int main(int argc, char *argv[]) return 1; } - if (!(ring.features & IORING_FEAT_CQE_SKIP)) { - printf("IOSQE_CQE_SKIP_SUCCESS is not supported, skip\n"); - return 0; - } + if (!(ring.features & IORING_FEAT_CQE_SKIP)) + return T_EXIT_SKIP; for (i = 0; i < 4; i++) { bool skip_last = i & 1; diff --git a/contrib/libs/liburing/test/socket.c b/contrib/libs/liburing/test/socket.c index c3250c8dc0..b4514e3154 100644 --- a/contrib/libs/liburing/test/socket.c +++ b/contrib/libs/liburing/test/socket.c @@ -273,7 +273,6 @@ static int do_send(int socket_direct, int alloc) } if (cqe->res < 0) { if (cqe->res == -EINVAL) { - fprintf(stdout, "No socket support, skipping\n"); no_socket = 1; io_uring_cqe_seen(&ring, cqe); return fallback_send(&ring, &saddr); diff --git a/contrib/libs/liburing/test/sq-poll-dup.c b/contrib/libs/liburing/test/sq-poll-dup.c index bbeb63e6cb..a821f21962 100644 --- a/contrib/libs/liburing/test/sq-poll-dup.c +++ b/contrib/libs/liburing/test/sq-poll-dup.c @@ -172,14 +172,21 @@ int main(int argc, char *argv[]) vecs = t_create_buffers(BUFFERS, BS); fd = open(fname, O_RDONLY | O_DIRECT); - if (fname != argv[1]) - unlink(fname); - if (fd < 0) { + int __e = errno; + + if (fname != argv[1]) + unlink(fname); + + if (__e == EINVAL) + return T_EXIT_SKIP; perror("open"); return -1; } + if (fname != argv[1]) + unlink(fname); + ret = test(fd, 0, 0); if (ret) { fprintf(stderr, "test 0 0 failed\n"); diff --git a/contrib/libs/liburing/test/sq-poll-kthread.c b/contrib/libs/liburing/test/sq-poll-kthread.c index 4ec43a9323..f4d4801816 100644 --- a/contrib/libs/liburing/test/sq-poll-kthread.c +++ b/contrib/libs/liburing/test/sq-poll-kthread.c @@ -118,7 +118,7 @@ err_pipe: return ret; } -int test_sq_poll_kthread_stopped(bool do_exit) +static int test_sq_poll_kthread_stopped(bool do_exit) { pid_t pid; int status = 0; diff --git a/contrib/libs/liburing/test/sq-poll-share.c b/contrib/libs/liburing/test/sq-poll-share.c index a2af97543f..c8a72107f9 100644 --- a/contrib/libs/liburing/test/sq-poll-share.c +++ b/contrib/libs/liburing/test/sq-poll-share.c @@ -90,13 +90,14 @@ int main(int argc, char *argv[]) vecs = t_create_buffers(BUFFERS, BS); fd = open(fname, O_RDONLY | O_DIRECT); - if (fname != argv[1]) - unlink(fname); if (fd < 0) { perror("open"); return -1; } + if (fname != argv[1]) + unlink(fname); + for (i = 0; i < NR_RINGS; i++) { struct io_uring_params p = { }; diff --git a/contrib/libs/liburing/test/sqpoll-cancel-hang.c b/contrib/libs/liburing/test/sqpoll-cancel-hang.c index 81a30e27de..302a662bcd 100644 --- a/contrib/libs/liburing/test/sqpoll-cancel-hang.c +++ b/contrib/libs/liburing/test/sqpoll-cancel-hang.c @@ -10,8 +10,16 @@ #include <time.h> #include <unistd.h> #include "liburing.h" +#include "helpers.h" #include "../src/syscall.h" +/* + * This syzbot test is known broken on some archs, just allow the ones that + * are regularly tested. + */ +#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ + defined(__aarch64__) + static uint64_t current_time_ms(void) { struct timespec ts; @@ -40,7 +48,7 @@ static void kill_and_wait(int pid, int* status) #define WAIT_FLAGS __WALL -uint64_t r[3] = {0xffffffffffffffff, 0x0, 0x0}; +static uint64_t r[3] = {0xffffffffffffffff, 0x0, 0x0}; static long syz_io_uring_setup(volatile long a0, volatile long a1, volatile long a2, volatile long a3, volatile long a4, volatile long @@ -93,7 +101,7 @@ SIZEOF_IO_URING_CQE + 63) & ~63; } -void trigger_bug(void) +static void trigger_bug(void) { intptr_t res = 0; *(uint32_t*)0x20000204 = 0; @@ -134,7 +142,7 @@ void trigger_bug(void) } int main(void) { - mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); + mmap((void *)0x20000000ul, 0x1000000ul, 7ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); int pid = fork(); if (pid < 0) exit(1); @@ -153,6 +161,9 @@ int main(void) } return 0; } - - - +#else +int main(void) +{ + return T_EXIT_SKIP; +} +#endif diff --git a/contrib/libs/liburing/test/sqpoll-disable-exit.c b/contrib/libs/liburing/test/sqpoll-disable-exit.c index b2a4160c58..6e5bfc6f47 100644 --- a/contrib/libs/liburing/test/sqpoll-disable-exit.c +++ b/contrib/libs/liburing/test/sqpoll-disable-exit.c @@ -137,7 +137,7 @@ static void kill_and_wait(int pid, int* status) } } -static void setup_test() +static void setup_test(void) { prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); setpgrp(); @@ -189,9 +189,9 @@ void execute_one(void) } int main(void) { - mmap((void *)0x1ffff000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); - mmap((void *)0x20000000ul, 0x1000000ul, 7ul, 0x32ul, -1, 0ul); - mmap((void *)0x21000000ul, 0x1000ul, 0ul, 0x32ul, -1, 0ul); + mmap((void *)0x1ffff000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); + mmap((void *)0x20000000ul, 0x1000000ul, 7ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); + mmap((void *)0x21000000ul, 0x1000ul, 0ul, MAP_ANON|MAP_PRIVATE, -1, 0ul); loop(); return 0; } diff --git a/contrib/libs/liburing/test/symlink.c b/contrib/libs/liburing/test/symlink.c index 755b51d152..1fc6b8e1db 100644 --- a/contrib/libs/liburing/test/symlink.c +++ b/contrib/libs/liburing/test/symlink.c @@ -44,7 +44,8 @@ err: return 1; } -int test_link_contents(const char* linkname, const char *expected_contents) +static int test_link_contents(const char* linkname, + const char *expected_contents) { char buf[128]; int ret = readlink(linkname, buf, 127); diff --git a/contrib/libs/liburing/test/test.h b/contrib/libs/liburing/test/test.h index 3628163afe..e99a8d204d 100644 --- a/contrib/libs/liburing/test/test.h +++ b/contrib/libs/liburing/test/test.h @@ -14,7 +14,8 @@ typedef struct io_uring_test_config { const char *description; } io_uring_test_config; -io_uring_test_config io_uring_test_configs[] = { +__attribute__((__unused__)) +static io_uring_test_config io_uring_test_configs[] = { { 0, "default" }, { IORING_SETUP_SQE128, "large SQE"}, { IORING_SETUP_CQE32, "large CQE"}, diff --git a/contrib/libs/liburing/test/timeout-new.c b/contrib/libs/liburing/test/timeout-new.c index d4aeced46b..e0f2935e63 100644 --- a/contrib/libs/liburing/test/timeout-new.c +++ b/contrib/libs/liburing/test/timeout-new.c @@ -13,9 +13,9 @@ #define TIMEOUT_MSEC 200 #define TIMEOUT_SEC 10 -int thread_ret0, thread_ret1; -int cnt = 0; -pthread_mutex_t mutex; +static int thread_ret0, thread_ret1; +static int cnt = 0; +static pthread_mutex_t mutex; static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec) { @@ -112,7 +112,8 @@ static int test_return_after_timeout(struct io_uring *ring) return 0; } -int __reap_thread_fn(void *data) { +static int __reap_thread_fn(void *data) +{ struct io_uring *ring = (struct io_uring *)data; struct io_uring_cqe *cqe; struct __kernel_timespec ts; @@ -124,12 +125,14 @@ int __reap_thread_fn(void *data) { return io_uring_wait_cqe_timeout(ring, &cqe, &ts); } -void *reap_thread_fn0(void *data) { +static void *reap_thread_fn0(void *data) +{ thread_ret0 = __reap_thread_fn(data); return NULL; } -void *reap_thread_fn1(void *data) { +static void *reap_thread_fn1(void *data) +{ thread_ret1 = __reap_thread_fn(data); return NULL; } @@ -138,7 +141,8 @@ void *reap_thread_fn1(void *data) { * This is to test issuing a sqe in main thread and reaping it in two child-thread * at the same time. To see if timeout feature works or not. */ -int test_multi_threads_timeout() { +static int test_multi_threads_timeout(void) +{ struct io_uring ring; int ret; bool both_wait = false; diff --git a/contrib/libs/liburing/test/timeout-overflow.c b/contrib/libs/liburing/test/timeout-overflow.c deleted file mode 100644 index 8c57ff31b1..0000000000 --- a/contrib/libs/liburing/test/timeout-overflow.c +++ /dev/null @@ -1,205 +0,0 @@ -#include "../config-host.h" -/* SPDX-License-Identifier: MIT */ -/* - * Description: run timeout overflow test - * - */ -#include <errno.h> -#include <stdio.h> -#include <limits.h> -#include <string.h> -#include <sys/time.h> - -#include "liburing.h" -#include "helpers.h" - -#define TIMEOUT_MSEC 200 -static int not_supported; - -static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec) -{ - ts->tv_sec = msec / 1000; - ts->tv_nsec = (msec % 1000) * 1000000; -} - -static int check_timeout_support(void) -{ - struct io_uring_sqe *sqe; - struct io_uring_cqe *cqe; - struct __kernel_timespec ts; - struct io_uring_params p; - struct io_uring ring; - int ret; - - memset(&p, 0, sizeof(p)); - ret = io_uring_queue_init_params(1, &ring, &p); - if (ret) { - fprintf(stderr, "ring setup failed: %d\n", ret); - return T_EXIT_FAIL; - } - - /* not really a match, but same kernel added batched completions */ - if (p.features & IORING_FEAT_POLL_32BITS) { - not_supported = 1; - return T_EXIT_SKIP; - } - - sqe = io_uring_get_sqe(&ring); - msec_to_ts(&ts, TIMEOUT_MSEC); - io_uring_prep_timeout(sqe, &ts, 1, 0); - - ret = io_uring_submit(&ring); - if (ret < 0) { - fprintf(stderr, "sqe submit failed: %d\n", ret); - goto err; - } - - ret = io_uring_wait_cqe(&ring, &cqe); - if (ret < 0) { - fprintf(stderr, "wait completion %d\n", ret); - goto err; - } - - if (cqe->res == -EINVAL) { - not_supported = 1; - fprintf(stdout, "Timeout not supported, ignored\n"); - return 0; - } - - io_uring_cqe_seen(&ring, cqe); - io_uring_queue_exit(&ring); - return T_EXIT_PASS; -err: - io_uring_queue_exit(&ring); - return T_EXIT_FAIL; -} - -/* - * We first setup 4 timeout requests, which require a count value of 1, 1, 2, - * UINT_MAX, so the sequence is 1, 2, 4, 2. Before really timeout, this 4 - * requests will not lead the change of cq_cached_tail, so as sq_dropped. - * - * And before this patch. The order of this four requests will be req1->req2-> - * req4->req3. Actually, it should be req1->req2->req3->req4. - * - * Then, if there is 2 nop req. All timeout requests expect req4 will completed - * successful after the patch. And req1/req2 will completed successful with - * req3/req4 return -ETIME without this patch! - */ -static int test_timeout_overflow(void) -{ - struct io_uring_sqe *sqe; - struct io_uring_cqe *cqe; - struct __kernel_timespec ts; - struct io_uring ring; - int i, ret; - - ret = io_uring_queue_init(16, &ring, 0); - if (ret) { - fprintf(stderr, "ring setup failed: %d\n", ret); - return 1; - } - - msec_to_ts(&ts, TIMEOUT_MSEC); - for (i = 0; i < 4; i++) { - unsigned num = 0; - sqe = io_uring_get_sqe(&ring); - switch (i) { - case 0: - case 1: - num = 1; - break; - case 2: - num = 2; - break; - case 3: - num = UINT_MAX; - break; - } - io_uring_prep_timeout(sqe, &ts, num, 0); - } - - for (i = 0; i < 2; i++) { - sqe = io_uring_get_sqe(&ring); - io_uring_prep_nop(sqe); - io_uring_sqe_set_data(sqe, (void *) 1); - } - ret = io_uring_submit(&ring); - if (ret < 0) { - fprintf(stderr, "sqe submit failed: %d\n", ret); - goto err; - } - - i = 0; - while (i < 6) { - ret = io_uring_wait_cqe(&ring, &cqe); - if (ret < 0) { - fprintf(stderr, "wait completion %d\n", ret); - goto err; - } - - /* - * cqe1: first nop req - * cqe2: first timeout req, because of cqe1 - * cqe3: second timeout req because of cqe1 + cqe2 - * cqe4: second nop req - * cqe5~cqe6: the left three timeout req - */ - switch (i) { - case 0: - case 3: - if (io_uring_cqe_get_data(cqe) != (void *) 1) { - fprintf(stderr, "nop not seen as 1 or 2\n"); - goto err; - } - break; - case 1: - case 2: - case 4: - if (cqe->res == -ETIME) { - fprintf(stderr, "expected not return -ETIME " - "for the #%d timeout req\n", i - 1); - goto err; - } - break; - case 5: - if (cqe->res != -ETIME) { - fprintf(stderr, "expected return -ETIME for " - "the #%d timeout req\n", i - 1); - goto err; - } - break; - } - io_uring_cqe_seen(&ring, cqe); - i++; - } - - return 0; -err: - return 1; -} - -int main(int argc, char *argv[]) -{ - int ret; - - if (argc > 1) - return T_EXIT_SKIP; - - ret = check_timeout_support(); - if (ret == T_EXIT_FAIL) { - fprintf(stderr, "check_timeout_support failed: %d\n", ret); - return T_EXIT_FAIL; - } - - if (not_supported) - return T_EXIT_SKIP; - - ret = test_timeout_overflow(); - if (ret) { - fprintf(stderr, "test_timeout_overflow failed\n"); - return T_EXIT_FAIL; - } - - return T_EXIT_PASS; -} diff --git a/contrib/libs/liburing/test/timeout.c b/contrib/libs/liburing/test/timeout.c index 41d8627938..d43c15db17 100644 --- a/contrib/libs/liburing/test/timeout.c +++ b/contrib/libs/liburing/test/timeout.c @@ -15,12 +15,14 @@ #include <sys/types.h> #include <sys/stat.h> +#include "helpers.h" #include "liburing.h" #include "../src/syscall.h" #define TIMEOUT_MSEC 200 static int not_supported; static int no_modify; +static int no_multishot; static void msec_to_ts(struct __kernel_timespec *ts, unsigned int msec) { @@ -177,7 +179,7 @@ static int test_single_timeout_nr(struct io_uring *ring, int nr) goto err; } i++; - }; + } return 0; err: @@ -1304,7 +1306,6 @@ static int test_not_failing_links(void) fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); return 1; } else if (cqe->user_data == 1 && cqe->res == -EINVAL) { - fprintf(stderr, "ETIME_SUCCESS is not supported, skip\n"); goto done; } else if (cqe->res != -ETIME || cqe->user_data != 1) { fprintf(stderr, "timeout failed %i %i\n", cqe->res, @@ -1329,6 +1330,246 @@ done: } +static int test_timeout_multishot(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int ret; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_MULTISHOT); + io_uring_sqe_set_data(sqe, (void *) 1); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + for (int i = 0; i < 2; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + if (ret == -EINVAL) { + no_multishot = 1; + return T_EXIT_SKIP; + } + + if (!(cqe->flags & IORING_CQE_F_MORE)) { + fprintf(stderr, "%s: flag not set in cqe\n", __FUNCTION__); + goto err; + } + + if (ret != -ETIME) { + fprintf(stderr, "%s: Timeout: %s\n", __FUNCTION__, strerror(-ret)); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + } + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + io_uring_prep_timeout_remove(sqe, 1, 0); + io_uring_sqe_set_data(sqe, (void *) 2); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + if (ret < 0) { + fprintf(stderr, "%s: remove failed: %s\n", __FUNCTION__, strerror(-ret)); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + if (ret != -ECANCELED) { + fprintf(stderr, "%s: timeout canceled: %s %llu\n", __FUNCTION__, strerror(-ret), cqe->user_data); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + return 0; +err: + return 1; +} + + +static int test_timeout_multishot_nr(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int ret; + + if (no_multishot) + return T_EXIT_SKIP; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, TIMEOUT_MSEC); + io_uring_prep_timeout(sqe, &ts, 3, IORING_TIMEOUT_MULTISHOT); + io_uring_sqe_set_data(sqe, (void *) 1); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + for (int i = 0; i < 3; i++) { + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + if (i < 2 && !(cqe->flags & IORING_CQE_F_MORE)) { + fprintf(stderr, "%s: flag not set in cqe\n", __FUNCTION__); + goto err; + } + if (i == 3 && (cqe->flags & IORING_CQE_F_MORE)) { + fprintf(stderr, "%s: flag set in cqe\n", __FUNCTION__); + goto err; + } + + ret = cqe->res; + if (ret != -ETIME) { + fprintf(stderr, "%s: Timeout: %s\n", __FUNCTION__, strerror(-ret)); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + } + + msec_to_ts(&ts, 2 * TIMEOUT_MSEC); + ret = io_uring_wait_cqe_timeout(ring, &cqe, &ts); + if (ret != -ETIME) { + fprintf(stderr, "%s: wait completion timeout %s\n", __FUNCTION__, strerror(-ret)); + goto err; + } + + return 0; +err: + return 1; +} + + +static int test_timeout_multishot_overflow(struct io_uring *ring) +{ + struct io_uring_cqe *cqe; + struct io_uring_sqe *sqe; + struct __kernel_timespec ts; + int ret; + + if (no_multishot) + return T_EXIT_SKIP; + + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + msec_to_ts(&ts, 10); + io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_MULTISHOT); + io_uring_sqe_set_data(sqe, (void *) 1); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + if (ret != -ETIME) { + fprintf(stderr, "%s: Timeout: %s\n", __FUNCTION__, strerror(-ret)); + goto err; + } + + io_uring_cqe_seen(ring, cqe); + sleep(1); + + if (!((*ring->sq.kflags) & IORING_SQ_CQ_OVERFLOW)) { + goto err; + } + + /* multishot timer should be gone */ + sqe = io_uring_get_sqe(ring); + if (!sqe) { + fprintf(stderr, "%s: get sqe failed\n", __FUNCTION__); + goto err; + } + + io_uring_prep_timeout_remove(sqe, 1, 0); + + ret = io_uring_submit(ring); + if (ret <= 0) { + fprintf(stderr, "%s: sqe submit failed: %d\n", __FUNCTION__, ret); + goto err; + } + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret < 0) { + fprintf(stderr, "%s: wait completion %d\n", __FUNCTION__, ret); + goto err; + } + + ret = cqe->res; + io_uring_cqe_seen(ring, cqe); + if (ret != -ETIME) { + fprintf(stderr, "%s: remove failed: %d %s\n", __FUNCTION__, ret, strerror(-ret)); + goto err; + } + + return 0; +err: + return 1; +} + + int main(int argc, char *argv[]) { struct io_uring ring, sqpoll_ring; @@ -1421,6 +1662,40 @@ int main(int argc, char *argv[]) return ret; } + ret = test_timeout_multishot(&ring); + if (ret && ret != T_EXIT_SKIP) { + fprintf(stderr, "test_timeout_multishot failed\n"); + return ret; + } + + ret = test_timeout_multishot_nr(&ring); + if (ret && ret != T_EXIT_SKIP) { + fprintf(stderr, "test_timeout_multishot_nr failed\n"); + return ret; + } + + /* io_uring_wait_cqe_timeout() may have left a timeout, reinit ring */ + io_uring_queue_exit(&ring); + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + + ret = test_timeout_multishot_overflow(&ring); + if (ret && ret != T_EXIT_SKIP) { + fprintf(stderr, "test_timeout_multishot_overflow failed\n"); + return ret; + } + + /* io_uring_wait_cqe_timeout() may have left a timeout, reinit ring */ + io_uring_queue_exit(&ring); + ret = io_uring_queue_init(8, &ring, 0); + if (ret) { + fprintf(stderr, "ring setup failed\n"); + return 1; + } + ret = test_single_timeout_wait(&ring, &p); if (ret) { fprintf(stderr, "test_single_timeout_wait failed\n"); diff --git a/contrib/libs/liburing/test/unlink.c b/contrib/libs/liburing/test/unlink.c index b5094ccb59..95e613ae3f 100644 --- a/contrib/libs/liburing/test/unlink.c +++ b/contrib/libs/liburing/test/unlink.c @@ -1,7 +1,7 @@ #include "../config-host.h" /* SPDX-License-Identifier: MIT */ /* - * Description: run various nop tests + * Description: run various unlink tests * */ #include <errno.h> diff --git a/contrib/libs/liburing/test/version.c b/contrib/libs/liburing/test/version.c new file mode 100644 index 0000000000..b7fbd8420b --- /dev/null +++ b/contrib/libs/liburing/test/version.c @@ -0,0 +1,26 @@ +#include "../config-host.h" +/* SPDX-License-Identifier: MIT */ +/* + * Description: check version macros and runtime checks work + * + */ +#include "liburing.h" +#include "helpers.h" + +int main(int argc, char *argv[]) +{ + if (!IO_URING_CHECK_VERSION(io_uring_major_version(), io_uring_minor_version())) + return T_EXIT_FAIL; + + if (io_uring_major_version() != IO_URING_VERSION_MAJOR) + return T_EXIT_FAIL; + + if (io_uring_minor_version() != IO_URING_VERSION_MINOR) + return T_EXIT_FAIL; + +#if !IO_URING_CHECK_VERSION(IO_URING_VERSION_MAJOR, IO_URING_VERSION_MINOR) + return T_EXIT_FAIL; +#endif + + return T_EXIT_PASS; +} diff --git a/contrib/libs/liburing/test/version.t/ya.make b/contrib/libs/liburing/test/version.t/ya.make new file mode 100644 index 0000000000..236860709b --- /dev/null +++ b/contrib/libs/liburing/test/version.t/ya.make @@ -0,0 +1,31 @@ +# Generated by devtools/yamaker. + +PROGRAM() + +WITHOUT_LICENSE_TEXTS() + +PEERDIR( + contrib/libs/liburing +) + +ADDINCL( + contrib/libs/liburing/src/include +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +CFLAGS( + -DLIBURING_BUILD_TEST + -D__SANE_USERSPACE_TYPES__ +) + +SRCDIR(contrib/libs/liburing/test) + +SRCS( + helpers.c + version.c +) + +END() diff --git a/contrib/libs/liburing/test/wakeup-hang.c b/contrib/libs/liburing/test/wakeup-hang.c index 1d1140c15e..66d49ff873 100644 --- a/contrib/libs/liburing/test/wakeup-hang.c +++ b/contrib/libs/liburing/test/wakeup-hang.c @@ -6,10 +6,10 @@ #include <stdlib.h> #include <string.h> #include <pthread.h> -#include <liburing.h> #include <fcntl.h> #include <poll.h> #include <sys/time.h> +#include "liburing.h" struct thread_data { struct io_uring *ring; diff --git a/contrib/libs/liburing/test/xattr.c b/contrib/libs/liburing/test/xattr.c index e1f97e5829..0c5870d33d 100644 --- a/contrib/libs/liburing/test/xattr.c +++ b/contrib/libs/liburing/test/xattr.c @@ -52,10 +52,11 @@ static int io_uring_fsetxattr(struct io_uring *ring, int fd, const char *name, } ret = cqe->res; - if (ret == -EINVAL) - no_xattr = 1; + if (ret < 0) { + if (cqe->res == -EINVAL || cqe->res == -EOPNOTSUPP) + no_xattr = 1; + } io_uring_cqe_seen(ring, cqe); - return ret; } @@ -127,8 +128,11 @@ static int io_uring_setxattr(struct io_uring *ring, const char *path, } ret = cqe->res; + if (ret < 0) { + if (ret == -EINVAL || ret == -EOPNOTSUPP) + no_xattr = 1; + } io_uring_cqe_seen(ring, cqe); - return ret; } diff --git a/contrib/libs/liburing/ya.make b/contrib/libs/liburing/ya.make index ce8ba55e8e..97083fd647 100644 --- a/contrib/libs/liburing/ya.make +++ b/contrib/libs/liburing/ya.make @@ -1,4 +1,4 @@ -# Generated by devtools/yamaker from nixpkgs 22.05. +# Generated by devtools/yamaker from nixpkgs 22.11. LIBRARY() @@ -10,9 +10,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2.3) +VERSION(2.4) -ORIGINAL_SOURCE(https://github.com/axboe/liburing/archive/liburing-2.3.tar.gz) +ORIGINAL_SOURCE(https://github.com/axboe/liburing/archive/liburing-2.4.tar.gz) ADDINCL( GLOBAL contrib/libs/liburing/src/include @@ -27,10 +27,12 @@ CFLAGS( ) SRCS( + src/nolibc.c src/queue.c src/register.c src/setup.c src/syscall.c + src/version.c ) END() @@ -54,6 +56,7 @@ RECURSE( test/buf-ring.t test/ce593a6c480a.t test/close-opath.t + test/connect-rep.t test/connect.t test/cq-full.t test/cq-overflow.t @@ -68,10 +71,12 @@ RECURSE( test/drop-submit.t test/eeed8b54e0df.t test/empty-eownerdead.t + test/eploop.t test/eventfd-disable.t test/eventfd-reg.t test/eventfd-ring.t test/eventfd.t + test/evloop.t test/exec-target.t test/exit-no-cleanup.t test/fadvise.t @@ -95,6 +100,7 @@ RECURSE( test/io_uring_register.t test/io_uring_setup.t test/iopoll-leak.t + test/iopoll-overflow.t test/iopoll.t test/lfs-openat-write.t test/lfs-openat.t @@ -103,6 +109,8 @@ RECURSE( test/link_drain.t test/madvise.t test/mkdir.t + test/msg-ring-flags.t + test/msg-ring-overflow.t test/msg-ring.t test/multicqes_drain.t test/nolibc.t @@ -113,6 +121,7 @@ RECURSE( test/open-direct-pick.t test/openat2.t test/personality.t + test/pipe-bug.t test/pipe-eof.t test/pipe-reuse.t test/poll-cancel-all.t @@ -122,6 +131,8 @@ RECURSE( test/poll-many.t test/poll-mshot-overflow.t test/poll-mshot-update.t + test/poll-race-mshot.t + test/poll-race.t test/poll-ring.t test/poll-v-poll.t test/poll.t @@ -132,6 +143,9 @@ RECURSE( test/recv-msgall-stream.t test/recv-msgall.t test/recv-multishot.t + test/reg-hint.t + test/reg-reg-ring.t + test/regbuf-merge.t test/register-restrictions.t test/rename.t test/ring-leak.t @@ -174,10 +188,10 @@ RECURSE( test/teardowns.t test/thread-exit.t test/timeout-new.t - test/timeout-overflow.t test/timeout.t test/tty-write-dpoll.t test/unlink.t + test/version.t test/wakeup-hang.t test/xattr.t ) |